diff --git "a/wandb/run-20231118_132020-qg88991p/logs/debug-internal.log" "b/wandb/run-20231118_132020-qg88991p/logs/debug-internal.log" --- "a/wandb/run-20231118_132020-qg88991p/logs/debug-internal.log" +++ "b/wandb/run-20231118_132020-qg88991p/logs/debug-internal.log" @@ -33240,3 +33240,5368 @@ 2023-11-19 07:22:54,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive 2023-11-19 07:22:54,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status 2023-11-19 07:22:54,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:22:57,470 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:00,366 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:23:03,158 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:08,159 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:09,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:23:09,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:23:09,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:23:10,431 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:23:13,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:18,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:20,496 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:23:23,150 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:23:24,151 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:24,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:23:24,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:23:24,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:23:28,543 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:23:29,958 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:34,959 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:38,606 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:23:39,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:23:39,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:23:39,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:23:40,419 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:45,420 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:46,664 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:23:51,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:53,151 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:23:54,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:23:54,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:23:54,390 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:23:56,488 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:23:56,728 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:01,489 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:06,602 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:06,793 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:09,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:24:09,389 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:24:12,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:13,545 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:24:13,546 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:24:13,547 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:24:13,550 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:24:13,834 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:24:14,838 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:15,842 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:18,552 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:23,152 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:24:24,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:24,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:24:24,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:24:24,389 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:24:24,897 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:29,636 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:33,014 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:34,916 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:39,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:24:39,756 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:24:39,758 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:24:40,008 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:43,084 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:45,981 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:51,150 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:24:51,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:24:53,152 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:24:54,756 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:24:54,756 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:24:54,757 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:24:56,937 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:01,221 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:25:02,201 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:07,236 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:09,756 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:25:09,757 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:25:11,282 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:25:12,988 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:17,989 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:19,334 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:25:23,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:23,153 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:25:26,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:25:26,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:25:26,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:25:28,202 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:29,399 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:25:33,203 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:38,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:39,480 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:25:41,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:25:41,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:25:44,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:47,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:25:49,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:53,154 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:25:55,168 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:25:56,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:25:56,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:25:56,611 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:26:00,348 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:04,661 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:26:05,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:10,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:11,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:26:11,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:26:14,723 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:26:16,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:21,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:23,155 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:26:24,790 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:26:26,121 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:26:26,121 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:26:26,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:31,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:34,857 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:26:36,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:41,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:26:41,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:26:41,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:26:42,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:42,922 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:26:47,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:52,409 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:26:52,984 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:26:53,156 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:26:56,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:26:56,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:26:57,413 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:02,779 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:03,048 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:27:07,781 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:11,105 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:27:11,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:27:11,259 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:27:13,478 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:18,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:19,175 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:27:19,176 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:27:19,177 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:27:19,178 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:27:20,176 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:27:20,177 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:27:21,181 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:27:23,156 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:27:24,156 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:27,278 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:27:27,279 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:27:27,280 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:27:30,139 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:31,238 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:27:35,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:40,209 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:41,319 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:27:42,278 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:27:42,279 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:27:42,279 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:27:45,468 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:50,749 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:51,386 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:27:53,156 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:27:56,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:27:57,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:27:57,281 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:27:57,282 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:27:59,441 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:28:02,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:07,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:09,500 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:28:12,279 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:28:12,279 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:28:12,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:17,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:19,559 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:28:22,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:23,157 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:28:27,279 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:28:27,280 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:28:27,634 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:28:28,444 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:33,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:37,703 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:28:38,759 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:42,281 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:28:42,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:28:42,284 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:28:44,487 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:47,764 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:28:50,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:53,158 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:28:55,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:28:55,819 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:28:57,279 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:28:57,280 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:29:00,463 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:04,879 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:29:06,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:11,373 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:12,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:29:12,281 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:29:12,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:29:12,942 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:29:16,473 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:21,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:23,014 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:29:23,158 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:29:27,159 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:27,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:29:27,280 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:29:31,075 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:29:32,810 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:37,811 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:41,143 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:29:42,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:29:42,280 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:29:43,524 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:48,526 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:51,202 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:29:53,169 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:29:54,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:29:57,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:29:57,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:29:57,550 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:29:59,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:01,264 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:04,651 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:09,315 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:10,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:12,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:30:12,549 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:30:15,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:19,381 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:21,042 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:23,159 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:30:26,161 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:26,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:30:26,260 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:30:26,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:30:26,262 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:30:26,431 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:30:27,437 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:27,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:30:27,549 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:30:28,442 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:31,784 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:37,504 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:37,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:42,614 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:43,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:30:43,626 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:30:43,627 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:30:47,572 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:47,784 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:52,786 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:53,160 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:30:55,634 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:30:58,372 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:30:58,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:30:58,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:30:58,625 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:31:03,833 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:05,700 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:31:08,895 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:13,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:31:13,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:31:13,626 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:31:14,811 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:15,760 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:31:19,813 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:23,161 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:31:23,810 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:31:25,161 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:30,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:30,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:31:30,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:31:30,223 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:31:33,874 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:31:35,499 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:40,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:41,934 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:31:45,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:31:45,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:31:45,222 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:31:46,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:51,562 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:31:52,002 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:31:53,161 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:31:57,162 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:00,068 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:32:00,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:32:00,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:32:00,222 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:32:02,469 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:08,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:09,126 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:32:13,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:15,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:32:15,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:32:15,857 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:32:19,059 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:19,192 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:32:23,163 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:32:24,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:27,247 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:32:29,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:30,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:32:30,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:32:30,857 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:32:35,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:37,305 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:32:40,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:45,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:32:45,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:32:45,857 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:32:46,063 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:47,372 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:32:51,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:32:53,164 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:32:55,418 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:32:57,040 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:01,963 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:33:01,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:33:01,967 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:33:02,198 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:05,483 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:33:07,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:12,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:15,546 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:33:16,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:33:16,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:33:16,965 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:33:18,154 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:23,164 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:33:23,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:25,607 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:33:28,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:31,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:33:31,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:33:31,965 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:33:32,412 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:33:32,414 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:33:32,414 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:33:32,419 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:33:32,680 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:33:33,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:33,685 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:33:34,690 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:33:38,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:43,781 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:33:44,067 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:46,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:33:46,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:33:46,965 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:33:49,136 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:53,166 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:33:53,834 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:33:54,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:33:59,168 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:01,889 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:34:01,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:34:01,965 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:34:05,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:10,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:15,699 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:15,966 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:34:16,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:34:16,965 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:34:21,208 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:23,166 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:34:26,033 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:34:27,136 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:32,137 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:32,188 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:34:32,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:34:32,191 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:34:36,091 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:34:37,545 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:42,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:44,142 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:34:47,188 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:34:47,189 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:34:48,443 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:53,167 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:34:54,168 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:34:54,214 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:34:59,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:02,189 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:35:02,189 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:35:03,489 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:35:04,434 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:09,436 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:11,538 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:35:14,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:18,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:35:18,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:35:18,440 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:35:20,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:21,718 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:35:23,167 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:35:26,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:29,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:35:31,879 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:33,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:35:33,439 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:35:37,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:39,846 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:35:42,833 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:48,222 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:48,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:35:48,439 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:35:49,913 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:35:53,168 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:35:54,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:35:57,970 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:35:59,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:03,441 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:36:03,442 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:36:03,443 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:36:05,643 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:08,025 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:36:11,213 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:16,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:18,092 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:36:18,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:36:18,440 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:36:22,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:23,170 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:36:26,137 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:36:27,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:32,749 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:33,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:36:33,441 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:36:36,196 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:36:38,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:43,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:43,172 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:36:43,173 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:36:43,173 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:36:43,175 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:36:43,251 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:36:45,263 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:36:46,268 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:36:48,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:48,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:36:48,441 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:36:53,170 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:36:54,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:36:54,315 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:36:59,172 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:03,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:37:03,441 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:37:04,379 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:37:04,669 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:09,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:12,435 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:37:14,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:18,843 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:37:18,845 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:37:18,846 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:37:19,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:21,492 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:37:23,170 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:37:25,171 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:30,527 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:31,563 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:37:33,844 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:37:33,844 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:37:33,845 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:37:36,090 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:39,617 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:37:41,453 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:46,454 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:48,844 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:37:48,844 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:37:48,845 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:37:49,763 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:37:52,080 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:53,170 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:37:57,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:37:57,819 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:38:02,378 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:04,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:38:04,178 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:38:04,179 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:38:07,883 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:38:08,271 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:13,273 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:15,944 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:38:18,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:19,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:38:19,177 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:38:23,172 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:38:23,636 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:26,020 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:38:28,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:33,891 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:34,080 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:38:34,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:38:34,177 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:38:39,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:44,212 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:38:44,836 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:49,838 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:38:49,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:38:49,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:38:49,875 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:38:52,283 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:38:53,172 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:38:55,173 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:00,174 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:02,344 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:39:04,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:39:04,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:39:04,875 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:39:06,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:11,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:12,414 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:39:16,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:19,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:39:19,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:39:19,874 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:39:22,071 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:22,477 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:39:23,173 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:39:27,175 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:30,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:39:32,487 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:34,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:39:34,876 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:39:34,877 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:39:38,057 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:40,591 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:39:43,250 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:48,075 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:39:48,076 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:39:48,076 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:39:48,079 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:39:48,638 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:39:49,080 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:49,642 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:39:49,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:39:49,875 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:39:50,645 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:39:53,173 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:39:54,174 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:59,381 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:39:59,708 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:40:04,382 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:05,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:40:05,657 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:40:05,658 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:40:07,766 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:40:09,782 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:14,783 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:17,834 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:40:20,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:20,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:40:20,656 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:40:23,175 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:40:25,712 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:25,893 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:40:30,713 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:35,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:40:35,656 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:40:35,903 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:35,978 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:40:40,904 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:44,031 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:40:46,183 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:51,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:40:51,787 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:40:51,788 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:40:51,789 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:40:53,175 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:40:54,098 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:40:57,176 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:02,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:04,164 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:41:06,787 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:41:06,788 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:41:06,788 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:41:08,032 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:12,230 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:41:13,303 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:18,304 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:21,787 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:41:21,788 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:41:22,290 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:41:23,175 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:41:24,176 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:29,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:30,346 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:41:34,295 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:37,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:41:37,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:41:37,609 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:41:39,751 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:40,433 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:41:44,752 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:50,311 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:50,495 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:41:52,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:41:52,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:41:52,607 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:41:53,176 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:41:56,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:41:58,545 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:01,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:07,125 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:07,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:42:07,607 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:42:07,607 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:42:08,615 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:12,778 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:18,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:18,673 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:22,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:42:22,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:42:22,611 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:42:23,190 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:42:23,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:28,574 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:28,736 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:33,576 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:36,799 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:37,607 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:42:37,607 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:42:37,608 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:42:38,785 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:43,787 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:46,867 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:48,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:53,062 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:42:53,063 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:42:53,064 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:42:53,235 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:42:54,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:42:54,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:42:54,266 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:42:54,267 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:42:54,268 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:42:54,915 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:42:55,919 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:56,923 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:42:59,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:03,967 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:43:05,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:08,062 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:43:08,062 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:43:08,063 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:43:10,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:14,037 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:43:16,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:21,107 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:22,090 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:43:23,062 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:43:23,063 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:43:23,302 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:43:26,303 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:31,526 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:32,155 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:43:36,527 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:38,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:43:38,717 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:43:38,719 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:43:41,762 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:42,221 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:43:46,763 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:50,287 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:43:52,267 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:43:53,180 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:43:53,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:43:53,717 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:43:57,901 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:00,363 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:44:03,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:08,618 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:08,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:44:08,717 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:44:10,426 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:44:13,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:18,471 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:44:19,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:23,181 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:44:24,425 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:44:24,426 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:44:24,427 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:44:24,647 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:26,529 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:44:30,504 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:36,137 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:36,587 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:44:39,425 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:44:39,426 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:44:41,633 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:46,657 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:44:47,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:52,541 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:44:53,181 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:44:54,426 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:44:54,426 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:44:54,427 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:44:54,710 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:44:57,673 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:03,485 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:04,780 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:45:08,485 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:10,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:45:10,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:45:10,559 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:45:12,837 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:45:13,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:18,613 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:22,903 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:45:23,182 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:45:24,183 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:25,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:45:25,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:45:25,558 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:45:29,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:32,972 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:45:34,839 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:40,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:45:40,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:45:40,558 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:45:40,819 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:41,035 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:45:45,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:51,055 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:51,101 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:45:53,182 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:45:56,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:45:56,143 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:45:56,144 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:45:56,145 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:45:57,285 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:45:57,286 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:45:57,286 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:45:57,288 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:45:58,145 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:45:59,151 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:00,156 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:01,290 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:07,218 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:08,205 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:11,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:46:11,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:46:11,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:46:12,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:16,276 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:17,436 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:22,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:23,183 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:46:26,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:46:26,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:46:26,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:46:26,353 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:27,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:32,442 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:36,421 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:38,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:42,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:46:42,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:46:42,132 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:46:44,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:44,480 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:49,306 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:53,184 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:46:54,545 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:46:55,184 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:46:57,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:46:57,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:46:57,131 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:47:00,390 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:02,607 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:47:05,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:10,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:12,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:47:12,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:47:12,131 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:47:12,669 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:47:16,311 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:21,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:22,727 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:47:23,184 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:47:27,186 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:28,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:47:28,292 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:47:28,294 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:47:30,811 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:47:32,461 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:37,462 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:40,889 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:47:42,576 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:43,290 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:47:43,292 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:47:48,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:50,953 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:47:53,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:53,185 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:47:58,186 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:47:58,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:47:58,291 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:48:03,276 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:05,023 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:48:08,277 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:13,074 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:48:13,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:48:13,292 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:48:13,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:18,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:23,141 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:48:23,185 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:48:24,186 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:29,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:48:29,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:48:29,014 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:48:29,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:33,221 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:48:34,384 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:39,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:43,291 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:48:44,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:48:44,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:48:44,013 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:48:45,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:50,594 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:51,344 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:48:53,186 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:48:56,187 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:48:59,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:48:59,014 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:48:59,015 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:49:00,414 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:01,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:06,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:08,695 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:49:08,696 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:49:08,697 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:49:08,701 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:49:09,467 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:49:09,468 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:10,472 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:11,703 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:14,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:49:14,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:49:14,013 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:49:17,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:20,533 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:23,187 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:49:23,188 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:28,599 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:29,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:49:29,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:49:29,014 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:49:29,198 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:34,200 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:38,664 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:39,599 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:44,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:44,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:49:44,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:49:44,689 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:49:46,717 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:49,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:53,188 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:49:55,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:49:56,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:49:59,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:49:59,686 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:50:00,892 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:06,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:06,844 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:50:11,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:14,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:50:14,687 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:50:14,896 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:50:16,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:21,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:23,188 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:50:24,962 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:50:27,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:30,206 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:50:30,207 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:50:30,209 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:50:32,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:33,014 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:50:37,397 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:42,952 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:43,083 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:50:45,206 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:50:45,207 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:50:45,207 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:50:48,454 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:51,140 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:50:53,189 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:50:54,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:50:59,982 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:00,206 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:51:00,207 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:51:00,207 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:51:01,211 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:51:05,412 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:09,302 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:51:11,069 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:15,779 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:51:15,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:51:15,782 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:51:16,945 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:19,366 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:51:21,958 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:23,191 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:51:27,100 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:27,429 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:51:30,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:51:30,780 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:51:32,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:37,500 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:51:38,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:43,588 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:45,779 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:51:45,780 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:51:47,569 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:51:49,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:53,193 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:51:54,877 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:51:55,624 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:51:59,879 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:01,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:52:01,721 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:52:01,723 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:52:04,685 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:52:05,781 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:10,782 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:11,699 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:52:11,701 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:52:11,701 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:52:11,705 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:52:11,736 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:52:12,741 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:52:13,747 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:52:16,707 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:16,721 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:52:16,721 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:52:16,721 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:52:22,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:22,813 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:52:23,193 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:52:28,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:30,864 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:52:31,721 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:52:31,721 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:52:33,968 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:38,970 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:40,917 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:52:44,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:47,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:52:47,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:52:47,216 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:52:50,079 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:51,010 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:52:53,194 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:52:55,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:52:59,073 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:53:01,088 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:02,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:53:02,215 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:53:06,420 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:09,141 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:53:12,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:17,192 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:53:17,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:53:17,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:17,215 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:53:22,468 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:23,195 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:53:27,253 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:53:27,838 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:32,840 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:33,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:53:33,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:53:33,730 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:53:37,317 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:53:37,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:42,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:47,387 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:53:48,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:48,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:53:48,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:53:48,729 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:53:53,195 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:53:54,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:53:55,445 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:53:59,560 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:03,504 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:54:03,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:54:03,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:54:03,730 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:54:04,999 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:10,000 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:13,566 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:54:15,986 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:19,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:54:19,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:54:19,938 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:54:20,997 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:23,196 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:54:23,625 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:54:26,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:31,854 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:33,696 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:54:34,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:54:34,937 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:54:34,938 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:54:37,161 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:41,760 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:54:42,883 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:47,885 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:49,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:54:49,937 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:54:49,937 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:54:51,824 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:54:53,154 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:53,207 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:54:58,968 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:54:59,877 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:03,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:05,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:55:05,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:55:05,852 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:55:08,937 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:09,881 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:14,883 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:15,770 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:55:15,772 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:55:15,772 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:55:15,775 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:55:15,992 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:55:16,999 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:18,005 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:20,777 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:20,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:55:20,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:55:20,851 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:55:23,198 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:55:26,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:27,060 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:31,198 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:35,123 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:35,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:55:35,851 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:55:37,067 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:42,068 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:45,183 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:47,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:51,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:55:51,230 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:55:51,232 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:55:53,198 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:55:53,198 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:55:53,231 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:55:58,199 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:03,289 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:56:03,290 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:06,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:56:06,230 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:56:08,414 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:13,347 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:56:13,751 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:18,753 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:21,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:56:21,230 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:56:21,401 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:56:23,198 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:56:24,199 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:29,979 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:31,463 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:56:34,980 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:36,872 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:56:36,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:56:36,875 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:56:39,519 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:56:40,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:45,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:49,586 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:56:51,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:51,872 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:56:51,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:56:51,873 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:56:53,199 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:56:56,581 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:56:57,638 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:57:01,582 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:06,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:57:06,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:57:06,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:06,875 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:57:07,724 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:57:12,057 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:17,256 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:17,782 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:57:22,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:23,200 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:57:23,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:57:23,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:57:23,368 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:57:25,841 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:57:27,484 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:32,486 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:35,914 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:57:38,173 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:38,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:57:38,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:57:38,366 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:57:43,417 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:45,975 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:57:48,418 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:53,200 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:57:53,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:57:53,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:57:53,366 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:57:53,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:57:56,033 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:57:58,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:03,613 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:08,614 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:09,483 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:58:09,484 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:58:13,711 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:18,712 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:21,207 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:58:23,201 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:58:24,202 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:24,482 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:58:24,483 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:58:29,480 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:31,269 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:58:34,481 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:38,588 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 07:58:38,590 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 07:58:38,590 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 07:58:38,593 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 07:58:39,317 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 07:58:39,322 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:58:39,483 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:58:39,484 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:58:39,682 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:40,324 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:58:44,683 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:49,381 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:58:50,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:53,201 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:58:54,483 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:58:54,484 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:58:55,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:58:59,444 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:59:00,854 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:05,855 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:09,483 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:59:09,484 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:59:09,522 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:59:11,647 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:17,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:17,584 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:59:22,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:23,203 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:59:24,622 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 07:59:24,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:59:24,626 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:59:27,643 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:59:27,733 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:32,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:35,697 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:59:37,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:39,622 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:59:39,623 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:59:43,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:45,754 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:59:49,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:53,203 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 07:59:54,622 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 07:59:54,623 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 07:59:54,818 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 07:59:55,812 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 07:59:59,818 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:03,865 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:00:05,090 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:10,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:11,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:00:11,092 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:00:11,094 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:00:13,925 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:00:15,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:20,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:23,204 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:00:23,993 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:00:26,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:00:26,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:00:26,092 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:26,092 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:00:32,060 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:00:32,081 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:37,082 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:41,099 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:00:41,100 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:00:41,100 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:00:42,133 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:00:42,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:47,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:52,201 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:00:52,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:00:53,204 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:00:56,099 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:00:56,100 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:00:56,100 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:00:58,315 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:02,261 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:03,596 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:08,602 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:11,099 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:01:11,100 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:01:11,101 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:01:12,319 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:14,251 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:19,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:20,379 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:23,215 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:01:25,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:26,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:01:26,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:01:29,441 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:30,302 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:35,303 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:37,504 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:40,392 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:41,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:01:41,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:01:45,504 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:45,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:01:45,571 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:01:45,571 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:01:45,574 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:01:46,564 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:01:46,564 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:47,570 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:50,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:53,206 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:01:55,624 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:01:56,037 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:01:56,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:01:56,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:02:01,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:06,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:09,708 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:02:11,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:02:11,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:02:11,500 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:16,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:21,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:23,206 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:02:27,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:02:27,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:02:27,107 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:02:27,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:32,351 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:37,352 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:42,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:02:42,104 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:02:43,287 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:48,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:53,207 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:02:53,993 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:02:54,207 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:02:57,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:02:57,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:02:59,294 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:02,047 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:03:05,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:10,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:12,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:03:12,108 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:03:12,108 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:03:12,112 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:03:15,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:21,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:22,185 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:03:23,208 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:03:26,209 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:27,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:03:27,932 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:03:27,935 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:03:30,247 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:03:31,273 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:36,274 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:40,323 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:03:41,458 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:42,930 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:03:42,931 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:03:47,181 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:50,394 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:03:52,757 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:53,208 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:03:57,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:03:57,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:03:57,931 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:03:58,457 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:04:03,188 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:08,526 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:04:08,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:12,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:04:12,931 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:04:14,168 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:18,593 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:04:20,087 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:23,209 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:04:25,640 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:04:25,970 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:27,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:04:27,932 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:04:31,189 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:35,699 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:04:37,133 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:42,134 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:42,932 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:04:42,933 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:04:42,934 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:04:45,761 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:04:47,662 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:52,663 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:53,209 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:04:53,819 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:04:57,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:04:57,932 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:04:57,932 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:05:02,984 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:03,880 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:07,984 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:11,925 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:12,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:05:12,932 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:05:13,178 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:18,180 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:21,991 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:23,210 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:05:23,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:28,212 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:28,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:05:28,536 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:05:28,537 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:05:29,656 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:05:29,658 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:05:29,658 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:05:29,661 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:05:30,037 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:05:31,044 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:32,048 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:33,663 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:38,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:40,129 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:43,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:05:43,535 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:05:43,733 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:48,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:50,227 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:53,210 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:05:54,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:05:58,302 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:05:58,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:05:58,535 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:05:59,783 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:04,785 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:08,368 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:06:10,158 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:13,950 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:06:13,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:06:13,952 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:06:16,109 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:16,420 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:06:21,109 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:23,212 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:06:26,480 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:06:26,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:28,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:06:28,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:06:28,952 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:06:32,137 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:36,536 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:06:37,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:42,858 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:43,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:06:43,951 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:06:44,591 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:06:48,200 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:53,212 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:06:53,213 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:06:54,649 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:06:58,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:00,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:07:00,502 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:07:00,503 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:07:02,714 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:07:03,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:08,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:12,803 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:07:14,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:15,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:07:15,502 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:07:19,701 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:22,871 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:07:23,213 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:07:25,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:29,923 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:07:30,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:07:30,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:07:30,502 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:07:30,502 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:35,751 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:37,983 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:07:41,711 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:45,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:07:45,581 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:07:45,583 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:07:46,821 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:48,064 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:07:51,822 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:53,214 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:07:57,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:07:58,131 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:00,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:08:00,581 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:08:02,745 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:06,185 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:08,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:13,067 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:15,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:08:15,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:08:15,596 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:08:16,303 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:18,766 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:23,214 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:08:23,976 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:26,364 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:28,977 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:31,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:08:31,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:08:31,949 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:08:33,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:08:33,167 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:08:33,168 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:08:33,170 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:08:33,490 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:08:34,172 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:34,493 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:35,498 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:39,173 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:44,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:44,560 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:46,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:08:46,948 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:08:46,948 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:08:50,156 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:08:52,623 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:08:53,215 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:08:55,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:01,072 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:01,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:09:01,947 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:09:02,684 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:09:06,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:10,736 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:09:11,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:16,434 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:17,356 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:09:17,358 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:09:17,359 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:09:20,804 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:09:22,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:23,216 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:09:27,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:28,856 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:09:32,356 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:09:32,356 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:09:32,357 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:09:32,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:38,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:38,922 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:09:43,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:46,976 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:09:47,356 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:09:47,356 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:09:47,357 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:09:48,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:53,217 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:09:54,219 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:09:57,045 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:09:59,390 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:03,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:10:03,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:10:03,587 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:10:04,771 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:07,108 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:10:09,933 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:14,161 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:10:15,885 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:18,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:10:18,586 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:10:21,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:23,217 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:10:24,219 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:10:27,218 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:32,219 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:33,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:10:33,586 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:10:34,294 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:10:37,832 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:42,343 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:10:43,523 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:48,525 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:48,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:10:48,589 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:10:48,590 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:10:52,412 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:10:53,218 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:10:54,218 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:10:59,220 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:02,477 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:03,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:11:03,587 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:11:04,751 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:10,527 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:10,726 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:15,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:18,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:11:18,588 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:11:20,600 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:20,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:23,218 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:11:26,219 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:28,646 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:31,668 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:33,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:11:33,587 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:11:36,744 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:37,054 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:11:37,055 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:11:37,056 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:11:37,059 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:11:37,699 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:11:38,703 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:39,707 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:42,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:47,241 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:48,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:11:48,587 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:11:48,762 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:52,827 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:11:53,220 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:11:56,808 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:11:58,148 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:03,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:03,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:12:03,588 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:12:04,853 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:12:08,212 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:13,213 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:14,968 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:12:18,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:12:18,587 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:12:18,816 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:23,220 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:12:24,220 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:25,035 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:12:29,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:33,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:12:33,587 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:12:34,830 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:35,126 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:12:39,832 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:43,179 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:12:45,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:49,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:12:49,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:12:49,370 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:12:50,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:12:53,220 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:12:53,240 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:12:56,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:01,391 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:03,312 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:13:04,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:13:04,366 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:13:06,562 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:11,364 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:13:12,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:17,519 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:19,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:13:19,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:13:19,368 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:13:20,431 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:13:22,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:23,221 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:13:28,222 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:30,503 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:13:33,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:34,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:13:34,366 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:13:38,563 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:13:39,384 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:44,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:48,626 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:13:49,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:13:49,375 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:13:49,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:53,222 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:13:55,225 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:13:56,674 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:00,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:04,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:14:04,367 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:14:05,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:06,732 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:10,945 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:14,782 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:16,346 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:19,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:14:19,367 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:14:21,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:23,233 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:14:24,855 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:27,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:32,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:32,902 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:34,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:14:34,367 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:14:37,570 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:41,819 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:14:41,820 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:14:41,821 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:14:41,826 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:14:41,964 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:14:42,828 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:42,971 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:43,976 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:47,829 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:49,629 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:14:49,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:14:49,631 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:14:53,038 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:14:53,223 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:14:53,224 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:14:58,225 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:01,144 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:15:03,717 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:04,629 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:15:04,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:15:04,630 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:15:08,786 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:11,210 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:15:13,986 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:19,333 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:15:19,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:15:19,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:15:19,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:19,631 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:15:23,223 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:15:25,225 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:29,397 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:15:30,588 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:35,397 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:15:35,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:15:35,401 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:15:35,589 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:39,458 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:15:41,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:46,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:47,508 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:15:50,397 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:15:50,397 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:15:52,627 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:15:53,225 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:15:57,568 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:15:57,708 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:02,709 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:05,398 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:16:05,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:16:05,400 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:16:08,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:10,650 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:16:13,588 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:18,704 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:16:18,799 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:20,397 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:16:20,398 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:16:23,225 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:16:24,226 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:28,774 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:16:29,759 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:34,761 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:35,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:16:35,705 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:16:35,706 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:16:38,837 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:16:39,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:44,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:46,897 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:16:49,847 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:50,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:16:50,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:16:50,705 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:16:53,226 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:16:54,941 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:16:56,966 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:16:59,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:05,014 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:17:05,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:05,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:17:05,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:17:05,705 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:17:10,899 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:15,090 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:17:16,057 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:21,058 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:21,079 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:17:21,081 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:17:21,082 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:17:23,141 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:17:23,227 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:17:26,228 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:31,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:33,211 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:17:36,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:17:36,079 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:17:36,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:41,271 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:17:42,276 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:47,277 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:49,812 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:17:49,813 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:17:49,814 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:17:49,816 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:17:50,327 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:17:51,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:17:51,079 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:17:51,332 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:17:52,311 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:52,336 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:17:53,227 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:17:58,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:17:59,385 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:18:03,698 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:06,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:18:06,651 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:18:06,652 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:18:09,442 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:18:09,698 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:14,699 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:19,517 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:18:20,231 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:21,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:18:21,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:18:21,651 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:18:23,230 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:18:25,232 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:27,570 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:18:31,194 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:36,272 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:36,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:18:36,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:18:36,651 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:18:37,636 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:18:41,848 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:45,692 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:18:47,458 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:52,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:18:52,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:18:52,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:18:52,954 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:18:53,230 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:18:55,750 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:18:58,072 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:03,073 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:05,835 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:19:07,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:19:07,952 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:19:08,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:13,797 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:14,878 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:19:18,798 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:22,939 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:19:22,952 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:19:22,952 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:19:23,230 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:19:24,231 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:29,232 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:30,985 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:19:34,545 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:38,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:19:38,465 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:19:38,467 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:19:39,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:41,047 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:19:44,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:49,100 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:19:50,484 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:53,231 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:19:53,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:19:53,465 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:19:55,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:19:59,167 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:00,695 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:05,696 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:07,216 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:08,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:20:08,465 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:20:11,677 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:17,273 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:17,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:22,540 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:23,232 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:20:23,465 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:20:23,466 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:20:23,468 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:20:25,328 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:28,527 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:34,426 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:35,399 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:38,465 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:20:38,466 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:20:39,636 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:43,453 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:44,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:49,877 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:20:51,932 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:20:51,933 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:20:51,934 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:20:51,936 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:20:52,511 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:20:53,232 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:20:53,465 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:20:53,466 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:20:53,516 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:54,521 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:20:55,626 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:00,628 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:01,572 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:21:05,878 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:08,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:21:08,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:21:08,688 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:21:11,631 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:21:11,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:16,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:21,692 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:21:22,044 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:23,232 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:21:23,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:21:23,685 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:21:27,840 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:29,748 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:21:32,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:38,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:38,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:21:38,685 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:21:39,815 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:21:43,843 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:47,866 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:21:49,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:53,234 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:21:54,370 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:21:54,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:21:54,372 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:21:54,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:21:57,926 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:22:00,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:05,541 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:05,980 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:22:09,370 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:22:09,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:22:09,372 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:22:10,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:15,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:16,037 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:22:20,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:23,234 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:22:24,086 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:22:24,370 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:22:24,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:22:24,371 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:22:25,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:30,627 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:33,145 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:22:36,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:40,046 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:22:40,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:22:40,049 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:22:42,112 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:43,217 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:22:47,113 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:51,274 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:22:52,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:22:53,234 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:22:55,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:22:55,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:22:55,048 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:22:58,207 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:01,343 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:23:04,199 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:09,393 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:23:09,831 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:10,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:23:10,047 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:23:15,297 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:19,455 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:23:20,821 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:23,235 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:23:26,236 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:26,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:23:26,536 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:23:26,537 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:23:29,526 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:23:31,871 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:36,872 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:37,579 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:23:41,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:23:41,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:23:41,535 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:23:42,717 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:47,644 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:23:48,049 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:53,050 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:53,235 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:23:57,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:23:57,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:23:57,387 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:23:58,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:23:58,792 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:23:58,793 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:23:58,793 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:23:58,796 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:23:59,718 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:23:59,719 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:00,724 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:03,798 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:08,799 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:11,789 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:12,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:24:12,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:24:12,386 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:24:14,589 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:19,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:21,861 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:23,236 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:24:25,237 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:27,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:24:27,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:24:27,386 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:24:30,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:33,936 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:36,086 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:41,087 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:42,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:24:42,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:24:42,386 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:24:44,008 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:46,017 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:46,835 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:51,837 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:53,236 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:24:54,071 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:24:57,237 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:24:57,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:24:57,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:24:57,387 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:25:03,199 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:04,130 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:25:08,200 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:12,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:25:12,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:25:12,387 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:25:13,581 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:14,194 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:25:18,581 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:23,237 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:25:24,237 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:24,282 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:25:27,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:25:27,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:25:27,387 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:25:29,631 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:34,350 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:25:34,891 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:39,892 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:42,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:25:42,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:25:42,388 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:25:43,402 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:25:45,590 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:50,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:51,465 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:25:53,237 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:25:56,238 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:25:57,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:25:57,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:25:57,388 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:26:01,533 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:26:01,849 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:06,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:11,602 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:26:12,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:26:12,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:12,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:26:12,388 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:26:17,668 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:21,689 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:26:23,238 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:26:23,248 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:28,249 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:28,465 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:26:28,466 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:26:28,468 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:26:31,753 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:26:33,552 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:38,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:39,798 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:26:43,465 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:26:43,465 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:26:43,466 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:26:43,712 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:49,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:49,947 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:26:53,238 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:26:55,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:26:58,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:26:58,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:27:00,019 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:27:00,715 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:05,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:10,086 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:27:11,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:13,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:27:13,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:27:16,858 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:17,748 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:27:17,749 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:27:17,750 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:27:17,752 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:27:18,140 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:27:20,150 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:27:22,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:23,239 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:27:28,201 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:27:28,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:28,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:27:28,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:27:33,725 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:38,259 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:27:39,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:43,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:27:43,481 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:27:44,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:48,317 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:27:50,603 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:53,241 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:27:56,244 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:27:58,382 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:27:58,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:27:58,477 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:28:01,642 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:06,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:08,446 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:28:11,705 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:14,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:28:14,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:28:14,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:28:16,525 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:28:17,295 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:22,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:23,243 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:28:26,593 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:28:28,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:29,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:28:29,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:28:29,102 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:28:33,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:35,654 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:28:38,626 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:43,712 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:28:44,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:28:44,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:28:44,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:44,102 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:28:49,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:53,244 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:28:53,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:28:55,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:28:59,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:28:59,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:28:59,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:29:00,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:03,844 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:29:05,753 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:10,754 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:11,900 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:29:14,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:29:14,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:29:16,304 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:21,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:21,962 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:29:23,245 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:29:27,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:29,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:29:29,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:29:29,104 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:29:30,006 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:29:32,406 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:37,408 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:40,070 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:29:42,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:44,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:29:44,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:29:48,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:50,145 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:29:53,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:53,245 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:29:58,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:29:59,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:29:59,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:30:03,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:04,226 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:30:08,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:13,904 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:14,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:30:14,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:30:14,291 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:30:19,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:22,351 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:30:23,246 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:30:25,247 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:29,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:30:29,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:30:29,369 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:30:30,566 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:30,924 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:30:30,925 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:30:30,926 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:30:30,936 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:30:31,413 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:30:32,418 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:30:35,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:41,467 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:42,493 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:30:44,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:30:44,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:30:44,367 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:30:46,617 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:51,924 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:52,561 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:30:53,247 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:30:57,248 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:30:59,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:30:59,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:30:59,377 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:31:00,617 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:31:02,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:07,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:10,674 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:31:13,052 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:14,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:31:14,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:31:14,377 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:31:18,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:20,740 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:31:23,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:23,247 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:31:28,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:28,794 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:31:29,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:31:29,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:31:29,377 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:31:33,627 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:37,859 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:31:39,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:44,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:31:44,378 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:31:44,380 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:31:44,632 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:47,912 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:31:50,406 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:53,248 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:31:55,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:31:55,962 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:31:59,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:31:59,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:31:59,378 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:32:01,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:06,099 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:32:06,562 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:11,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:14,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:32:14,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:32:14,378 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:32:16,160 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:32:17,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:22,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:23,249 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:32:24,386 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:32:28,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:29,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:32:29,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:32:29,378 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:32:33,894 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:34,443 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:32:38,895 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:44,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:44,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:32:44,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:32:44,378 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:32:44,578 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:32:49,540 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:32:53,249 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:32:54,640 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:32:55,251 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:00,253 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:00,321 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:33:00,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:33:00,324 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:33:04,714 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:33:05,896 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:10,897 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:14,780 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:33:15,321 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:33:15,322 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:33:15,322 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:33:16,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:22,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:22,834 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:33:23,249 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:33:27,251 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:30,322 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:33:30,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:33:30,325 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:33:32,907 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:33:32,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:37,952 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:40,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:33:40,302 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:33:40,303 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:33:40,305 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:33:40,958 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:33:42,969 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:33:43,307 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:45,322 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:33:45,322 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:33:45,323 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:33:48,486 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:52,035 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:33:53,250 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:33:54,251 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:33:59,252 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:00,322 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:34:00,322 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:34:02,099 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:34:04,530 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:10,164 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:34:10,232 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:15,233 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:15,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:34:15,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:34:15,325 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:34:20,282 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:34:20,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:23,250 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:34:26,252 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:30,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:34:30,323 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:34:30,343 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:34:31,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:36,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:38,431 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:34:42,030 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:45,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:34:45,324 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:34:47,156 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:48,501 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:34:52,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:34:53,252 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:34:56,560 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:34:57,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:00,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:35:00,323 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:35:03,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:06,619 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:35:09,235 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:15,208 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:15,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:35:15,324 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:35:16,690 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:35:20,527 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:23,252 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:35:24,746 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:35:26,253 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:31,070 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:35:31,071 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:35:31,072 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:35:32,139 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:34,808 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:35:37,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:42,858 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:35:42,982 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:46,070 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:35:46,070 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:35:48,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:35:52,952 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:35:53,253 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:35:54,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:00,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:01,009 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:01,070 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:36:01,071 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:36:05,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:11,074 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:11,184 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:16,088 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:36:16,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:36:16,090 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:36:16,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:19,126 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:22,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:23,253 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:36:27,363 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:29,190 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:31,087 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:36:31,088 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:36:33,253 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:38,280 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:38,461 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:43,463 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:46,088 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:36:46,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:36:46,090 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:36:46,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:36:46,797 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:36:46,798 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:36:46,801 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:36:47,353 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:36:48,348 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:48,802 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:49,351 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:53,254 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:36:54,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:36:58,406 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:36:59,929 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:01,088 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:37:01,089 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:37:05,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:08,472 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:37:10,890 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:16,088 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:37:16,089 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:37:16,269 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:16,527 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:37:21,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:23,256 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:37:26,589 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:37:26,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:31,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:37:31,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:37:31,093 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:37:32,352 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:34,640 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:37:38,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:43,027 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:44,713 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:37:46,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:37:46,089 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:37:48,287 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:53,256 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:37:54,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:37:54,772 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:37:59,268 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:01,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:38:01,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:38:01,390 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:38:04,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:04,839 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:38:09,458 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:12,895 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:38:14,661 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:16,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:38:16,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:38:16,388 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:38:20,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:22,958 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:38:23,256 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:38:26,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:31,012 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:38:31,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:38:31,388 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:38:31,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:36,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:41,087 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:38:41,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:46,745 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:47,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:38:47,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:38:47,106 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:38:51,159 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:38:52,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:53,257 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:38:58,137 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:38:59,223 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:02,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:39:02,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:39:02,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:39:03,311 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:09,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:09,295 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:14,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:17,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:39:17,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:39:17,355 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:19,315 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:23,270 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:39:25,090 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:27,419 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:30,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:32,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:39:32,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:39:32,682 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:39:35,468 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:35,822 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:40,823 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:45,529 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:46,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:47,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:39:47,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:39:47,682 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:39:51,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:39:52,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:39:52,327 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:39:52,327 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:39:52,329 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:39:52,567 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:39:53,258 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:39:54,576 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:55,581 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:39:57,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:02,316 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:02,630 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:40:02,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:40:02,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:40:02,682 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:40:07,940 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:12,701 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:40:13,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:18,576 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:18,636 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:40:18,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:40:18,639 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:40:20,758 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:40:23,259 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:40:24,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:29,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:30,816 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:40:33,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:40:33,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:40:33,638 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:40:34,822 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:40,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:40,872 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:40:45,352 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:48,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:40:48,638 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:40:48,926 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:40:50,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:53,259 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:40:56,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:40:58,984 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:41:01,485 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:04,921 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:41:04,924 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:41:04,926 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:41:07,022 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:07,045 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:41:12,024 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:17,228 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:41:17,949 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:19,922 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:41:19,922 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:41:19,923 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:41:23,094 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:23,260 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:41:27,291 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:41:28,582 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:33,583 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:34,922 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:41:34,923 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:41:34,923 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:41:35,339 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:41:39,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:44,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:45,407 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:41:49,667 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:41:50,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:41:50,966 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:41:50,968 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:41:53,260 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:41:53,470 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:41:55,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:00,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:03,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:42:05,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:42:05,966 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:05,966 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:42:05,966 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:42:11,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:11,594 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:42:16,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:20,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:42:20,966 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:42:21,716 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:42:22,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:23,261 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:42:27,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:28,757 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:42:33,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:36,898 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:42:36,899 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:42:36,900 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:42:38,070 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:38,820 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:42:43,071 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:48,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:48,881 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:42:51,899 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:42:51,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:42:51,926 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:42:53,261 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:42:54,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:42:56,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:42:56,906 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:42:56,907 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:42:56,908 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:42:57,052 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:42:59,066 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:42:59,910 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:04,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:06,899 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:43:06,900 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:43:06,900 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:43:07,129 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:43:10,076 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:15,791 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:17,205 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:43:20,793 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:22,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:43:22,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:43:22,626 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:43:23,262 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:43:25,263 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:43:25,829 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:30,830 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:35,322 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:43:36,321 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:37,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:43:37,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:43:37,625 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:43:41,781 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:45,401 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:43:47,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:52,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:43:52,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:43:52,625 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:43:53,262 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:43:58,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:03,264 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:07,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:44:07,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:44:07,626 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:44:08,787 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:13,788 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:18,789 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:22,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:44:22,625 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:44:23,263 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:44:23,858 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:44:24,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:29,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:32,907 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:44:34,858 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:37,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:44:37,626 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:44:39,878 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:42,976 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:44:45,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:50,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:44:52,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:44:52,626 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:44:53,037 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:44:53,263 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:44:56,264 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:01,902 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:03,104 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:45:06,903 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:08,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:45:08,622 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:45:08,624 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:45:11,160 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:45:12,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:17,782 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:21,218 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:45:23,263 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:45:23,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:23,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:45:23,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:45:23,622 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:45:28,708 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:29,272 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:45:33,709 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:38,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:45:38,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:45:38,622 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:45:38,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:39,346 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:45:43,877 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:49,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:49,408 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:45:53,277 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:45:53,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:45:53,622 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:45:53,623 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:45:54,831 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:45:57,461 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:46:00,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:05,306 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:07,526 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:46:08,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:46:08,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:46:08,622 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:46:10,821 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:16,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:17,595 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:46:21,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:23,266 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:46:23,623 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:46:23,624 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:46:23,625 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:46:25,648 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:46:27,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:32,256 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:33,234 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:46:33,235 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:46:33,236 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:46:33,238 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:46:33,717 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:46:35,727 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:46:38,240 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:38,621 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:46:38,622 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:46:43,837 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:45,790 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:46:48,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:53,266 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:46:53,622 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:46:53,622 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:46:53,850 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:46:54,792 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:46:59,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:01,908 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:47:05,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:09,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:47:09,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:47:09,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:47:11,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:11,983 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:47:16,401 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:21,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:22,043 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:47:23,267 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:47:24,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:47:24,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:47:24,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:47:26,481 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:31,744 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:32,118 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:47:36,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:39,168 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:47:39,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:47:39,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:47:42,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:47,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:49,227 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:47:52,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:53,267 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:47:55,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:47:55,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:47:55,603 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:47:58,762 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:47:59,279 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:48:03,764 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:07,330 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:48:08,786 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:10,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:48:10,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:48:10,600 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:48:13,787 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:15,383 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:48:19,691 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:23,268 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:48:25,451 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:48:25,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:48:25,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:48:25,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:25,601 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:48:30,854 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:35,514 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:48:36,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:40,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:48:40,602 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:48:40,604 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:48:42,798 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:43,562 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:48:47,799 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:53,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:48:53,268 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:48:53,626 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:48:55,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:48:55,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:48:55,602 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:48:58,845 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:01,680 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:49:04,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:10,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:10,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:49:10,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:49:10,601 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:49:11,753 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:49:15,832 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:19,805 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:49:21,414 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:23,269 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:49:26,383 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:49:26,384 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:49:26,385 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:49:26,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:29,878 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:49:31,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:36,607 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:36,751 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:49:36,752 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:49:36,753 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:49:36,756 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:49:36,925 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:49:37,930 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:49:41,383 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:49:41,383 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:49:42,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:47,712 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:48,000 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:49:52,714 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:49:53,270 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:49:56,062 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:49:56,383 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:49:56,383 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:49:58,633 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:03,634 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:06,129 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:50:08,702 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:12,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:50:12,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:50:12,551 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:50:13,732 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:16,190 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:50:18,943 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:23,270 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:50:24,238 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:50:24,271 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:27,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:50:27,549 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:50:29,818 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:34,310 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:50:35,074 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:40,075 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:42,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:50:42,549 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:50:44,381 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:50:45,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:50,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:53,271 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:50:56,272 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:50:57,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:50:57,550 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:51:01,714 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:06,715 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:11,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:12,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:51:12,549 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:51:16,733 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:21,790 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:51:21,949 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:23,271 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:51:27,295 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:27,688 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:51:27,690 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:51:27,692 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:51:29,855 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:51:33,006 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:38,007 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:39,924 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:51:42,688 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:51:42,689 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:51:43,906 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:48,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:53,281 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:51:54,150 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:51:54,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:51:57,688 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:51:57,689 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:51:59,886 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:04,887 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:09,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:13,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:52:13,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:52:13,797 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:52:15,043 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:18,286 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:52:20,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:23,273 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:52:25,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:26,338 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:52:28,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:52:28,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:52:28,747 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:52:30,948 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:36,000 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:36,392 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:52:41,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:43,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:52:43,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:52:43,747 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:52:44,453 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:52:46,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:51,946 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:53,273 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:52:54,516 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:52:57,274 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:52:58,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:52:58,749 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:52:58,750 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:53:02,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:04,579 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:53:07,717 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:13,500 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:13,659 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:53:13,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:53:13,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:53:13,748 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:53:18,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:21,710 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:53:23,274 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:53:24,275 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:28,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:53:28,748 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:53:28,750 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:53:29,943 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:29,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:53:29,946 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:53:29,946 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:53:29,949 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:53:30,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:53:31,780 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:53:34,951 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:40,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:41,844 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:53:43,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:53:43,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:53:43,748 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:53:45,993 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:49,907 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:53:51,774 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:53,274 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:53:57,276 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:53:58,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:53:58,748 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:53:58,749 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:53:59,978 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:54:02,950 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:08,048 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:54:08,684 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:13,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:14,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:54:14,458 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:54:18,152 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:54:18,714 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:23,275 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:54:24,277 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:28,223 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:54:29,430 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:29,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:54:29,457 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:54:34,648 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:38,296 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:54:40,473 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:44,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:54:44,456 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:54:45,616 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:46,342 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:54:50,618 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:53,276 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:54:54,397 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:54:56,361 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:54:59,458 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:54:59,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:54:59,461 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:55:01,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:04,472 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:55:07,471 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:12,974 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:14,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:55:14,457 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:55:14,533 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:55:18,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:23,276 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:55:24,277 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:24,602 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:55:29,279 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:29,457 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:55:29,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:55:29,459 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:55:32,666 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:55:34,527 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:40,521 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:42,742 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:55:44,457 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:55:44,457 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:55:45,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:50,798 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:55:51,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:53,277 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:55:57,278 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:55:57,848 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:55:59,457 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:55:59,457 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:56:02,684 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:07,915 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:56:08,402 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:13,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:14,941 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:56:14,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:56:14,943 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:56:18,053 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:56:19,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:23,278 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:56:24,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:26,118 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:56:29,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:56:29,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:56:29,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:29,943 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:56:34,595 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:56:34,596 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:56:34,597 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:56:34,598 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:56:35,170 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:56:35,599 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:36,176 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:56:40,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:44,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:56:44,942 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:56:46,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:46,240 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:56:51,167 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:53,278 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:56:54,306 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:56:56,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:56:59,943 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:56:59,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:56:59,945 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:57:02,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:04,375 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:57:07,961 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:13,889 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:14,431 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:57:14,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:57:14,943 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:57:19,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:22,480 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:57:23,279 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:57:24,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:29,281 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:29,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:57:29,943 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:57:32,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:57:35,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:40,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:42,600 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:57:44,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:57:44,943 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:57:45,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:50,972 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:52,675 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:57:53,279 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:57:56,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:57:59,943 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:57:59,943 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:58:00,725 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:58:02,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:07,159 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:10,784 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:58:12,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:15,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:58:15,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:58:15,774 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:58:18,767 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:18,830 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:58:23,280 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:58:24,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:26,890 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:58:29,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:30,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:58:30,773 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:58:34,998 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:36,953 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:58:40,175 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:45,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:45,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:58:45,773 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:58:47,015 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:58:50,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:58:53,291 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:58:56,293 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:00,102 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:59:00,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:59:00,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:59:00,773 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:59:02,030 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:07,032 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:08,152 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:59:12,642 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:15,774 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 08:59:15,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:59:15,777 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:59:17,997 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:18,217 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:59:22,997 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:23,282 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:59:28,281 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:59:28,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:30,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:59:30,774 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:59:33,979 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:38,344 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:59:39,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:44,020 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:45,006 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 08:59:45,007 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 08:59:45,007 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 08:59:45,010 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 08:59:45,391 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 08:59:45,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 08:59:45,774 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 08:59:46,395 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 08:59:49,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:53,283 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 08:59:55,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 08:59:56,498 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:00:00,346 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:02,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:00:02,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:00:02,134 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:00:04,553 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:00:06,316 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:11,317 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:14,624 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:00:16,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:17,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:00:17,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:00:17,132 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:00:22,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:23,284 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:00:24,695 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:00:27,286 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:32,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:00:32,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:00:32,133 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:00:32,293 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:34,771 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:00:37,294 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:43,155 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:44,876 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:00:47,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:00:47,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:00:47,131 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:00:48,294 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:52,936 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:00:53,284 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:00:54,285 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:00:59,287 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:02,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:01:02,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:01:02,132 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:01:03,018 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:01:04,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:09,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:13,077 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:01:14,616 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:17,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:01:17,133 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:01:17,135 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:01:19,930 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:21,138 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:01:23,285 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:01:25,286 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:30,361 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:01:30,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:32,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:01:32,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:01:32,132 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:01:36,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:40,433 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:01:41,488 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:46,490 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:47,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:01:47,133 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:01:47,134 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:01:48,614 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:01:52,052 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:53,286 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:01:57,176 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:01:58,675 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:02:02,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:02:02,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:02:02,133 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:02:02,316 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:07,505 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:08,855 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:02:12,507 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:16,908 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:02:17,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:02:17,133 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:02:17,133 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:02:18,392 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:23,287 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:02:24,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:26,977 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:02:29,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:32,415 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:02:32,415 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:02:32,417 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:02:35,028 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:02:35,462 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:40,463 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:45,091 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:02:46,297 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:47,415 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:02:47,416 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:02:51,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:02:51,246 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:02:51,247 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:02:51,249 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:02:52,139 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:02:52,250 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:02:53,144 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:02:53,287 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:02:57,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:02,415 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:03:02,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:02,416 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:03:03,392 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:03:07,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:10,454 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:03:13,506 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:17,420 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:03:17,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:03:17,424 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:03:18,631 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:20,528 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:03:23,287 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:03:24,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:28,580 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:03:29,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:32,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:03:32,421 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:03:34,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:38,651 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:03:39,664 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:44,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:46,710 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:03:47,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:03:47,422 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:03:50,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:53,290 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:03:56,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:03:56,773 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:04:01,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:03,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:04:03,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:04:03,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:04:04,833 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:04:06,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:11,179 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:14,899 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:04:17,175 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:18,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:04:18,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:04:22,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:23,290 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:04:24,954 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:04:28,292 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:33,009 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:04:33,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:04:33,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:04:33,146 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:04:33,313 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:38,314 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:43,067 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:04:43,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:48,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:04:48,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:04:48,107 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:04:49,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:04:53,138 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:04:53,290 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:04:55,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:00,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:01,236 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:03,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:05:03,104 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:05:06,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:11,307 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:11,937 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:16,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:18,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:05:18,106 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:05:19,363 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:22,332 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:23,291 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:05:28,314 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:29,432 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:33,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:05:33,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:05:33,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:38,771 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:39,498 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:43,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:47,552 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:48,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:05:48,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:05:49,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:53,294 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:05:54,954 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:05:54,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:05:54,957 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:05:54,958 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:05:54,960 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:05:55,599 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:05:56,603 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:57,607 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:05:59,962 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:03,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:06:03,106 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:06:04,654 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:06:05,220 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:10,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:14,741 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:06:15,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:18,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:06:18,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:06:21,269 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:23,293 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:06:24,806 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:06:26,294 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:31,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:33,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:06:33,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:06:34,874 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:06:36,329 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:41,331 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:42,937 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:06:46,717 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:48,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:06:48,108 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:06:52,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:06:53,008 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:06:53,293 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:06:57,294 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:01,061 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:07:02,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:03,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:07:03,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:07:08,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:11,124 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:07:14,206 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:18,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:07:18,107 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:07:19,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:21,187 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:07:23,294 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:07:25,295 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:29,238 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:07:30,519 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:33,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:07:33,107 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:07:36,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:39,304 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:07:41,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:46,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:47,363 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:07:48,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:07:48,106 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:07:52,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:07:53,296 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:07:57,425 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:07:57,441 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:02,442 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:03,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:08:03,106 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:08:08,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:13,325 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:17,633 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:08:18,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:08:18,107 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:08:18,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:23,295 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:08:24,297 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:27,700 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:08:29,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:34,346 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:34,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:08:34,450 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:08:34,452 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:08:37,769 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:08:39,657 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:44,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:47,828 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:08:49,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:08:49,450 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:08:49,450 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:08:49,689 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:08:53,297 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:08:54,869 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:08:55,498 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:00,499 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:04,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:09:04,450 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:09:04,451 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:09:04,937 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:09:05,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:10,614 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:12,825 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:09:12,826 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:09:12,826 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:09:12,828 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:09:13,002 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:09:13,002 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:09:15,014 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:09:15,829 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:20,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:09:20,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:09:20,737 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:09:20,988 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:23,077 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:09:23,297 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:09:26,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:31,945 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:33,175 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:09:35,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:09:35,735 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:09:37,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:41,230 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:09:43,525 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:48,526 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:50,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:09:50,736 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:09:51,283 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:09:53,298 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:09:54,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:09:59,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:01,347 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:10:04,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:07,225 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:10:07,226 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:10:07,227 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:10:09,406 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:10:10,325 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:15,327 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:19,483 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:10:21,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:22,224 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:10:22,225 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:10:23,298 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:10:27,295 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:27,536 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:10:32,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:37,224 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:10:37,225 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:10:37,382 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:37,600 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:10:42,383 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:45,651 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:10:47,884 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:52,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:10:53,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:10:53,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:10:53,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:10:53,336 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:10:55,705 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:10:58,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:03,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:05,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:11:08,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:11:08,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:11:08,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:11:09,322 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:13,832 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:11:14,760 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:19,761 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:23,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:11:23,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:11:23,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:11:23,325 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:11:23,901 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:11:25,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:30,327 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:31,978 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:11:35,659 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:38,767 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:11:38,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:11:38,770 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:11:40,890 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:42,056 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:11:45,890 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:51,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:51,122 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:11:53,300 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:11:53,767 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:11:53,768 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:11:56,946 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:11:59,183 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:12:01,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:06,993 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:08,768 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:12:08,768 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:12:09,258 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:12:12,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:16,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:12:16,918 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:12:16,919 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:12:16,921 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:12:17,323 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:12:17,324 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:12:18,923 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:19,336 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:12:23,302 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:12:24,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:25,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:12:25,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:12:25,147 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:12:27,397 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:12:29,443 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:34,462 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:39,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:40,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:12:40,146 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:12:41,483 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:12:44,693 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:49,695 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:51,536 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:12:53,302 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:12:55,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:12:55,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:12:55,146 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:12:59,601 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:13:00,987 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:05,988 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:09,660 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:13:10,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:13:10,146 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:13:11,390 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:16,391 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:19,722 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:13:22,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:23,305 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:13:25,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:13:25,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:13:25,148 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:13:27,273 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:27,777 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:13:32,274 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:35,826 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:13:38,168 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:40,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:13:40,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:13:40,146 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:13:43,391 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:45,901 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:13:48,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:53,305 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:13:53,776 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:13:55,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:13:55,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:13:55,147 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:13:55,969 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:13:59,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:04,036 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:14:04,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:09,689 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:10,887 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:14:10,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:14:10,889 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:14:14,110 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:14:15,029 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:20,031 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:23,305 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:14:24,176 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:14:25,306 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:25,887 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:14:25,888 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:14:30,694 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:32,231 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:14:35,695 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:40,836 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:40,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:14:40,935 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:14:40,936 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:14:42,300 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:14:46,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:49,350 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:14:51,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:53,305 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:14:56,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:14:56,645 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:14:56,646 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:14:56,859 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:14:59,439 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:15:02,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:07,492 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:15:08,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:11,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:15:11,645 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:15:13,898 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:17,583 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:15:19,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:23,307 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:15:24,309 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:25,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:15:25,559 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:15:25,559 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:15:25,561 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:15:25,646 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:15:26,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:15:26,645 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:15:27,659 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:15:29,819 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:35,537 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:35,718 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:15:40,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:42,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:15:42,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:15:42,262 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:15:45,782 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:15:46,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:51,395 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:15:53,307 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:15:53,838 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:15:57,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:15:57,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:15:57,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:15:57,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:02,460 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:03,904 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:16:07,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:12,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:16:12,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:16:12,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:16:13,506 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:13,977 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:16:18,507 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:22,033 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:16:23,308 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:16:24,309 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:27,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:16:27,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:16:27,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:16:29,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:32,096 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:16:34,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:40,151 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:16:40,304 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:42,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:16:42,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:16:42,268 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:16:45,523 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:50,261 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:16:50,812 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:53,308 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:16:56,310 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:16:57,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:16:57,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:16:57,264 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:17:00,333 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:17:01,967 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:06,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:07,571 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:17:12,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:12,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:17:12,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:17:12,262 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:17:17,489 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:17,641 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:17:22,490 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:23,310 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:17:27,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:17:27,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:17:27,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:17:27,497 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:27,706 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:17:32,498 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:35,757 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:17:37,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:42,592 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:17:42,592 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:17:42,592 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:17:42,840 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:45,820 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:17:48,669 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:53,310 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:17:53,887 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:17:54,311 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:17:57,592 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:17:57,593 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:17:59,760 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:03,959 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:18:05,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:11,529 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:12,013 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:18:12,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:18:12,619 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:18:16,878 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:21,977 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:22,090 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:18:23,311 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:18:27,312 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:27,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:18:27,777 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:18:27,779 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:18:28,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:18:28,996 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:18:28,996 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:18:28,997 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:18:29,175 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:18:30,180 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:18:32,999 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:38,000 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:40,247 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:18:42,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:18:42,776 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:18:42,777 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:18:43,979 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:48,300 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:18:49,000 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:53,314 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:18:54,316 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:18:57,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:18:57,776 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:18:58,365 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:18:59,943 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:05,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:06,416 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:19:10,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:13,545 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:19:13,546 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:19:16,476 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:19:16,739 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:21,740 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:23,313 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:19:26,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:19:27,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:28,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:19:28,546 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:19:32,726 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:34,591 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:19:37,909 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:43,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:19:43,547 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:19:43,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:44,650 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:19:48,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:52,707 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:19:53,313 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:19:54,314 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:19:59,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:19:59,143 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:19:59,144 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:19:59,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:02,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:20:05,203 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:10,827 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:20:11,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:14,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:20:14,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:20:16,361 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:19,878 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:20:21,450 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:23,314 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:20:27,315 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:27,930 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:20:29,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:20:29,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:20:32,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:37,998 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:20:38,286 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:43,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:45,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:20:45,164 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:20:45,166 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:20:48,065 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:20:49,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:53,316 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:20:54,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:20:56,128 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:20:59,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:00,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:21:00,164 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:21:05,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:06,192 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:21:10,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:15,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:21:15,164 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:21:15,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:16,255 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:21:20,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:23,316 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:21:24,305 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:21:25,626 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:30,164 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:21:30,165 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:21:31,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:33,045 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:21:33,046 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:21:33,046 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:21:33,048 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:21:33,372 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:21:34,379 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:21:37,049 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:42,051 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:44,450 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:21:45,164 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:21:45,164 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:21:47,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:52,507 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:21:52,510 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:21:53,316 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:21:58,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:00,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:22:00,166 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:22:02,579 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:22:03,503 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:08,506 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:10,646 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:22:14,452 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:15,164 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:22:15,165 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:22:20,094 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:20,720 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:22:23,317 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:22:25,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:30,164 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:22:30,165 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:22:30,392 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:30,789 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:22:35,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:40,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:40,850 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:22:45,536 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:46,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:22:46,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:22:46,552 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:22:48,917 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:22:50,646 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:53,317 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:22:56,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:22:58,992 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:23:01,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:23:01,550 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:23:01,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:23:01,774 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:06,065 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:23:06,791 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:11,793 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:16,133 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:23:16,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:23:16,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:23:16,551 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:23:16,813 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:21,814 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:23,318 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:23:26,209 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:23:27,138 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:31,976 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:23:31,977 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:23:31,978 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:23:32,231 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:34,271 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:23:38,161 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:42,350 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:23:44,023 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:46,975 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:23:46,976 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:23:49,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:52,407 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:23:53,319 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:23:54,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:23:59,321 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:01,976 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:24:01,976 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:24:01,977 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:24:02,506 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:05,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:10,454 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:10,577 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:15,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:17,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:24:17,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:24:17,401 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:24:20,518 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:20,647 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:23,319 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:24:26,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:28,704 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:31,469 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:32,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:24:32,400 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:24:36,646 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:37,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:24:37,146 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:24:37,147 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:24:37,148 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:24:37,771 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:24:37,772 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:38,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:42,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:47,223 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:47,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:24:47,400 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:24:48,835 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:52,590 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:24:53,320 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:24:56,886 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:24:58,113 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:03,114 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:03,553 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:25:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:25:03,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:25:06,957 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:25:08,701 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:13,702 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:17,025 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:25:18,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:25:18,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:25:18,555 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:25:18,766 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:23,321 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:25:24,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:25,085 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:25:29,020 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:33,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:25:33,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:25:33,555 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:25:34,779 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:35,153 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:25:39,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:43,208 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:25:45,099 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:48,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:25:48,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:25:48,558 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:25:50,759 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:25:53,271 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:25:53,321 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:25:56,322 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:01,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:03,330 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:26:03,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:26:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:26:03,555 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:26:06,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:11,770 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:16,425 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:26:17,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:18,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:26:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:26:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:26:22,822 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:23,324 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:26:26,557 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:26:28,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:33,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:26:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:26:33,555 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:26:33,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:34,619 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:26:38,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:42,676 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:26:44,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:26:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:26:48,557 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:26:49,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:26:52,749 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:26:53,324 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:26:55,325 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:00,327 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:02,811 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:27:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:27:03,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:27:05,800 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:10,874 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:11,034 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:16,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:27:18,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:27:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:27:20,933 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:21,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:23,325 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:27:27,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:28,991 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:32,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:27:33,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:27:33,557 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:27:37,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:39,064 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:42,996 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:45,975 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:27:45,977 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:27:45,978 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:27:45,980 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:27:46,116 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:27:47,121 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:48,126 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:27:48,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:27:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:27:48,764 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:53,327 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:27:54,329 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:27:57,184 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:27:59,881 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:03,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:28:03,954 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:28:03,956 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:28:05,174 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:07,254 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:28:10,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:15,304 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:15,317 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:28:18,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:28:18,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:28:18,954 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:28:21,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:23,326 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:28:25,386 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:28:26,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:31,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:33,441 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:28:33,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:28:33,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:28:33,954 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:28:37,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:42,247 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:43,507 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:28:47,249 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:49,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:28:49,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:28:49,107 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:28:50,557 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:28:52,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:28:53,329 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:28:57,330 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:00,622 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:29:03,155 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:04,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:29:04,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:29:04,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:29:08,672 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:29:09,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:14,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:18,738 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:29:19,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:29:19,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:29:19,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:29:19,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:23,330 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:29:25,331 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:28,798 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:29:31,321 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:35,122 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:29:35,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:29:35,124 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:29:36,436 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:36,844 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:29:41,437 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:46,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:46,904 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:29:50,122 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:29:50,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:29:50,123 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:29:52,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:29:53,331 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:29:56,978 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:29:57,961 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:02,963 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:05,031 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:30:05,122 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:30:05,124 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:30:05,124 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:30:08,303 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:13,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:15,084 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:30:18,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:20,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:30:20,125 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:30:20,126 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:30:23,331 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:30:24,427 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:25,146 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:30:29,428 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:35,043 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:35,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:30:35,124 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:30:35,124 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:30:35,302 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:30:40,317 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:43,362 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:30:45,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:50,567 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:30:51,114 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:30:51,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:30:51,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:30:52,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:30:52,255 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:30:52,256 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:30:52,258 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:30:52,422 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:30:53,336 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:30:53,425 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:30:54,429 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:30:56,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:01,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:03,490 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:31:06,114 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:31:06,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:31:06,115 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:31:07,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:13,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:13,549 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:31:18,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:21,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:31:21,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:31:21,116 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:31:23,343 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:31:23,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:23,617 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:31:25,231 ERROR gpu :60204 [interfaces.py:aggregate():161] Failed to serialize metric: The operating system has blocked the request. +2023-11-19 09:31:28,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:33,687 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:31:33,785 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:36,114 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:31:36,115 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:31:39,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:41,738 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:31:45,138 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:50,510 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:31:51,114 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:31:51,115 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:31:51,798 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:31:53,343 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:31:56,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:00,851 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:32:02,068 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:07,070 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:07,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:32:07,121 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:32:07,123 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:32:08,899 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:32:12,267 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:17,269 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:18,951 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:32:22,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:32:22,120 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:32:22,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:23,345 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:32:27,975 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:29,019 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:32:32,976 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:37,073 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:32:37,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:32:37,121 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:32:38,301 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:43,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:47,172 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:32:48,576 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:53,346 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:32:53,523 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:32:53,525 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:32:53,527 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:32:53,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:32:55,222 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:32:59,598 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:04,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:05,276 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:33:08,523 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:33:08,523 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:33:10,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:13,322 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:33:15,829 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:20,830 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:23,346 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:33:23,386 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:33:23,523 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:33:23,523 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:33:26,745 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:32,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:33,453 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:33:37,196 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:39,490 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:33:39,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:33:39,495 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:33:41,510 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:33:42,829 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:47,831 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:51,571 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:33:52,914 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:33:53,347 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:33:54,490 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:33:54,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:33:54,491 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:33:55,591 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:33:58,677 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:00,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:34:00,668 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:34:00,668 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:34:00,670 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:34:01,651 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:34:01,652 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:02,656 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:04,673 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:09,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:34:09,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:34:09,493 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:34:09,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:11,728 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:14,821 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:20,362 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:21,802 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:23,347 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:34:24,490 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:34:24,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:34:24,492 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:34:25,697 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:29,866 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:30,952 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:35,954 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:39,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:34:39,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:34:39,492 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:34:39,936 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:41,641 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:46,642 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:50,006 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:52,349 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:34:53,348 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:34:55,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:34:55,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:34:55,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:34:57,053 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:34:58,333 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:03,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:07,113 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:35:08,360 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:10,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:35:10,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:35:10,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:35:14,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:17,188 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:35:19,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:23,350 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:35:24,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:25,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:35:25,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:35:25,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:35:25,243 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:35:30,271 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:35,308 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:35:36,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:40,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:35:40,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:35:40,103 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:35:41,357 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:45,387 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:35:47,287 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:52,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:35:53,351 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:35:55,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:35:55,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:35:55,104 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:35:55,544 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:35:57,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:02,351 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:05,615 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:36:07,871 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:10,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:36:10,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:36:10,106 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:36:13,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:15,865 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:36:18,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:23,351 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:36:23,352 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:23,921 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:36:25,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:36:25,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:36:25,104 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:36:29,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:33,988 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:36:35,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:40,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:40,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:36:40,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:36:40,106 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:36:44,078 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:36:45,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:50,518 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:36:52,140 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:36:53,352 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:36:55,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:36:55,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:36:55,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:36:56,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:02,004 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:02,206 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:37:07,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:09,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:37:09,672 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:37:09,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:37:09,676 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:37:10,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:37:10,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:37:10,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:37:10,256 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:37:11,260 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:37:12,372 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:17,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:19,308 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:37:23,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:23,352 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:37:25,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:37:25,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:37:25,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:37:28,360 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:29,375 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:37:33,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:38,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:39,430 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:37:40,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:37:40,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:37:40,105 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:37:44,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:49,503 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:37:49,694 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:53,353 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:37:55,354 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:37:55,490 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:37:55,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:37:55,493 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:37:57,629 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:38:00,700 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:06,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:07,699 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:38:10,489 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:38:10,490 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:38:11,727 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:16,835 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:17,756 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:38:21,836 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:23,355 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:38:25,489 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:38:25,490 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:38:25,809 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:38:27,646 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:32,648 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:35,878 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:38:38,148 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:41,904 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:38:41,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:38:41,907 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:38:44,045 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:45,935 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:38:49,046 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:53,354 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:38:53,980 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:38:54,355 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:38:56,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:38:56,905 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:39:00,151 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:04,046 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:39:05,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:10,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:11,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:39:11,905 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:39:12,096 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:39:16,052 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:21,220 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:22,160 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:39:23,355 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:39:26,356 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:27,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:39:27,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:39:27,473 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:39:30,212 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:39:31,805 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:36,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:40,296 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:39:42,226 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:42,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:39:42,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:39:42,516 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:39:47,723 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:50,368 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:39:52,738 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:39:53,355 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:39:57,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:39:57,473 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:39:57,473 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:39:58,424 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:39:58,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:03,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:08,682 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:11,503 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:40:12,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:40:12,473 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:40:12,474 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:40:13,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:18,733 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:20,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:40:20,937 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:40:20,938 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:40:20,939 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:40:21,573 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:40:21,574 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:40:23,356 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:40:23,623 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:40:24,357 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:28,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:40:28,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:40:28,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:40:30,037 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:31,919 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:40:35,038 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:40,088 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:42,018 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:40:43,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:40:43,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:40:43,908 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:40:45,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:50,075 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:40:50,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:53,356 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:40:56,358 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:40:58,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:40:58,908 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:41:00,147 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:41:02,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:07,125 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:10,204 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:41:12,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:13,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:41:13,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:41:17,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:18,264 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:41:22,949 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:23,357 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:41:28,316 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:41:28,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:41:28,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:28,908 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:41:34,077 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:36,375 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:41:40,042 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:43,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:41:43,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:41:45,139 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:46,534 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:41:50,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:53,358 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:41:54,586 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:41:55,359 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:41:58,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:41:58,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:42:01,076 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:03,646 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:42:06,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:11,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:13,706 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:42:13,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:42:13,910 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:42:17,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:21,750 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:42:23,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:23,360 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:42:28,362 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:29,060 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:42:29,062 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:42:29,063 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:42:31,826 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:42:33,481 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:38,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:41,910 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:42:44,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:44,060 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:42:44,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:42:44,061 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:42:49,281 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:51,968 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:42:53,359 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:42:54,360 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:42:59,060 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:42:59,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:42:59,061 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:43:00,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:02,028 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:05,256 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:10,078 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:10,489 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:14,060 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:43:14,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:43:14,062 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:43:16,240 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:20,135 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:21,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:23,361 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:43:26,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:26,480 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:43:26,482 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:43:26,482 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:43:26,485 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:43:27,194 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:43:27,195 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:28,199 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:29,060 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:43:29,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:43:29,061 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:43:32,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:37,454 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:38,259 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:42,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:44,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:43:44,062 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:43:44,063 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:43:46,312 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:47,673 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:52,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:53,361 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:43:56,396 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:43:58,602 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:43:59,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:43:59,062 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:43:59,062 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:44:04,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:09,240 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:14,241 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:15,237 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:44:15,285 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:44:15,287 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:44:18,527 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:44:19,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:23,363 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:44:25,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:28,580 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:44:30,237 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:44:30,238 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:44:30,239 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:44:30,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:35,435 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:38,653 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:44:40,485 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:45,238 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:44:45,240 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:44:45,240 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:44:46,451 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:48,732 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:44:51,452 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:44:53,364 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:44:56,795 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:44:56,805 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:00,237 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:45:00,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:45:00,239 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:45:02,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:05,856 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:45:07,790 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:12,792 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:13,917 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:45:15,237 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:45:15,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:45:15,239 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:45:18,489 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:23,364 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:45:23,980 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:45:24,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:29,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:30,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:45:30,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:45:30,243 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:45:34,040 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:45:34,959 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:39,960 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:42,093 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:45:45,238 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:45:45,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:45,241 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:45:45,241 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:45:50,487 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:45:52,156 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:45:53,364 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:45:56,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:00,238 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:46:00,241 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:46:00,241 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:46:01,392 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:02,216 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:46:06,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:10,274 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:46:12,017 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:15,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:46:15,241 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:46:15,242 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:46:17,415 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:20,348 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:46:23,182 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:23,365 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:46:28,403 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:46:29,099 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:30,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:46:30,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:46:30,246 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:46:34,508 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:38,468 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:46:40,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:45,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:45,247 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:46:45,248 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:46:45,249 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:46:46,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:46:46,257 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:46:46,257 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:46:46,260 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:46:46,528 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:46:47,532 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:46:48,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:46:50,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:53,367 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:46:56,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:46:56,593 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:47:00,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:47:00,247 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:47:01,462 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:06,651 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:47:07,226 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:12,228 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:14,741 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:47:15,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:47:15,247 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:47:17,402 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:23,301 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:23,367 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:47:24,806 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:47:28,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:30,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:47:30,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:47:30,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:47:33,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:34,871 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:47:38,922 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:42,931 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:47:44,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:45,333 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:47:45,335 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:47:49,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:47:52,093 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:47:53,367 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:47:55,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:00,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:48:00,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:48:00,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:02,169 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:48:05,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:10,222 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:48:11,518 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:15,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:48:15,334 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:48:17,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:20,472 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:48:22,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:23,368 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:48:28,502 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:28,533 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:48:30,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:48:30,335 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:48:30,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:48:33,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:38,593 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:48:39,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:44,301 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:45,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:48:45,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:48:45,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:48:48,672 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:48:49,471 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:53,382 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:48:55,383 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:48:56,771 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:00,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:49:00,335 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:49:00,531 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:05,757 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:06,834 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:10,758 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:15,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:49:15,335 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:49:16,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:16,898 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:21,574 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:23,370 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:49:26,832 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:26,957 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:30,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:49:30,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:49:30,339 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:49:32,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:35,012 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:38,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:44,290 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:45,071 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:45,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:49:45,336 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:49:49,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:49:51,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 09:49:51,418 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 09:49:51,418 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 09:49:51,425 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 09:49:52,118 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 09:49:53,124 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:53,370 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:49:54,128 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:55,132 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 09:49:55,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:00,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:50:00,336 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:50:00,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:05,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:10,566 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:15,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:50:15,336 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:50:15,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:20,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:23,371 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:50:26,372 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:30,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:50:30,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:50:31,537 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:36,537 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:41,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:45,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:50:45,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:50:45,336 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:50:47,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:52,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:50:53,371 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:50:58,372 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:00,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:51:00,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:51:00,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:51:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:08,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:13,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:15,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:51:15,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:51:15,338 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:51:18,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:23,372 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:51:24,372 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:29,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:30,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:51:30,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:51:34,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:39,540 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:44,541 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:45,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:51:45,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:51:50,541 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:51:53,372 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:51:56,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:00,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:52:00,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:52:01,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:06,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:11,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:15,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:52:15,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:52:17,530 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:22,531 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:23,372 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:52:28,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:30,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:52:30,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:52:30,338 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:52:33,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:38,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:43,560 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:45,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:52:45,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:52:48,597 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:53,373 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:52:54,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:52:59,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:00,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:53:00,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:53:00,341 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:53:04,596 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:09,597 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:14,598 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:15,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:53:15,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:53:15,341 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:53:20,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:23,373 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:53:26,375 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:30,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:53:30,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:53:30,342 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:53:31,553 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:36,553 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:41,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:45,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:53:45,358 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:53:45,364 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:53:46,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:51,588 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:53:53,374 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:53:57,375 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:00,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:54:00,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:54:00,347 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:54:02,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:07,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:12,570 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:15,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:54:15,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:54:15,342 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:54:17,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:22,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:23,375 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:54:28,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:30,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:54:30,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:54:30,342 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:54:33,531 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:38,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:43,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:45,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:54:45,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:54:45,342 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:54:48,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:53,375 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:54:54,375 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:54:59,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:00,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:55:00,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:55:00,343 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:55:04,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:09,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:14,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:15,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:55:15,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:55:15,343 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:55:20,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:23,375 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:55:26,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:30,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:55:30,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:55:30,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:55:31,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:36,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:41,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:45,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:55:45,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:55:45,343 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:55:47,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:52,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:55:53,376 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:55:58,377 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:00,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:56:00,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:56:00,343 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:56:03,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:08,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:13,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:15,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:56:15,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:56:15,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:56:18,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:23,377 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:56:24,378 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:29,378 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:30,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:56:30,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:56:30,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:56:34,505 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:39,506 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:44,506 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:45,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:56:45,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:56:45,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:56:49,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:56:53,377 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:56:55,378 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:00,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:57:00,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:57:00,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:57:00,592 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:05,593 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:10,594 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:15,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:57:15,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:57:15,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:57:16,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:21,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:23,378 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:57:27,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:30,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:57:30,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:57:30,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:57:32,593 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:37,594 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:42,595 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:45,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:57:45,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:57:45,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:57:48,553 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:53,378 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:57:54,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:57:59,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:00,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:58:00,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:58:00,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:58:04,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:09,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:14,560 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:15,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:58:15,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:58:15,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:58:20,531 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:23,379 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:58:26,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:30,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:58:30,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:58:30,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:58:31,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:36,592 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:41,593 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:45,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:58:45,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:58:45,344 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:58:47,582 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:52,583 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:58:53,380 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:58:58,381 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:00,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:59:00,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:59:00,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:59:03,573 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:08,574 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:13,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:15,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:59:15,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:59:15,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:59:18,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:23,380 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:59:24,381 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:29,381 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:30,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:59:30,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:59:30,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:59:34,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:39,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:44,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:45,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 09:59:45,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 09:59:45,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 09:59:50,542 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 09:59:53,385 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 09:59:56,386 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:00,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:00:00,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:00:00,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:00:01,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:06,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:11,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:15,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:00:15,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:00:15,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:00:16,598 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:21,599 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:23,386 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:00:27,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:30,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:00:30,346 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:00:30,346 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:00:32,510 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:37,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:42,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:45,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:00:45,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:00:45,346 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:00:47,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:52,576 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:00:53,386 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:00:58,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:00,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 10:01:00,265 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 10:01:00,266 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 10:01:00,268 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 10:01:00,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:01:00,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:01:00,345 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:01:00,753 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 10:01:01,756 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:01:02,762 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:01:03,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:08,641 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:13,641 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:15,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:01:15,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:01:15,346 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:01:19,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:23,387 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:01:25,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:30,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:01:30,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:01:30,346 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:01:30,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:35,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:40,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:45,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:01:45,972 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:45,981 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:01:46,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:01:46,289 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:01:51,528 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:01:53,388 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:01:57,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:00,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:02:01,499 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:02:01,499 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:02:02,946 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:07,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:12,948 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:15,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:02:15,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:02:15,874 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:02:18,520 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:23,397 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:02:24,398 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:29,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:30,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:02:30,798 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:02:30,799 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:02:34,628 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:36,269 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:02:37,274 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:02:40,256 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:45,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:50,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:53,389 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:02:55,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:02:56,205 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:02:56,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 10:02:56,300 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-19 10:02:58,390 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:03:00,566 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:05,567 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:10,568 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:15,568 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:20,468 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:03:20,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:23,389 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:03:26,390 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:31,391 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:36,392 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:41,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:46,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:51,395 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:03:53,391 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:03:57,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:02,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:07,395 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:10,351 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:10,359 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:12,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:12,775 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:04:16,361 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:17,417 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:21,361 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:22,418 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:23,391 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:04:28,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:31,363 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:33,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:36,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:38,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:44,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:45,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:49,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:04:50,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:04:53,392 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:04:54,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:00,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:00,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:05,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:05,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:11,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:15,369 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:16,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:18,053 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:05:20,372 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:21,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:23,392 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:05:25,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:27,398 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:32,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:35,054 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:37,430 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:38,172 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:05:40,056 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:43,370 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:46,088 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:48,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:50,259 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:05:51,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:53,393 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:05:54,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:05:57,063 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:57,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:05:59,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:00,325 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:06:02,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:04,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:08,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:09,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:12,413 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:06:15,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:18,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:20,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:21,502 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:06:23,393 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:06:23,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:26,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:31,571 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:06:32,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:32,778 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:37,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:37,830 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:41,632 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:06:43,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:47,790 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:48,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:52,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:06:53,243 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:06:53,394 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:06:53,712 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:06:59,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:02,480 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:03,815 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:07:05,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:07,481 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:10,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:13,889 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:07:15,272 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:18,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:20,274 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:23,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:23,395 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:07:23,959 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:07:25,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:31,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:33,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:34,052 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:07:36,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:38,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:41,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:44,151 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:07:46,604 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:47,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:51,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:07:52,395 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:07:53,395 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:07:54,217 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:07:57,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:02,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:02,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:04,309 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:08:07,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:07,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:13,024 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:14,384 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:08:17,748 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:18,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:22,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:23,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:23,396 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:08:24,462 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:08:28,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:32,564 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:33,067 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:36,559 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:08:37,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:38,923 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:44,855 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:46,642 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:08:47,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:49,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:53,399 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:08:53,797 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:08:55,108 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:08:55,112 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 10:08:55,116 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 10:08:55,116 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 10:08:55,119 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 10:08:55,751 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 10:08:55,752 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:08:56,757 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:09:00,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:02,800 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:05,597 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:07,857 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:09:08,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:10,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:16,819 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:17,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:17,957 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:09:21,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:22,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:23,399 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:09:27,469 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:28,027 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:09:32,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:34,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:37,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:38,119 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:09:39,107 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:43,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:48,317 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:49,434 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:09:50,259 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:09:53,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:09:53,399 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:09:55,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:00,332 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:10:00,897 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:03,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:05,898 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:09,058 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:10,899 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:16,398 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:18,028 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:18,526 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:10:21,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:23,031 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:23,401 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:10:26,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:28,606 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:10:31,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:33,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:36,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:39,072 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:40,691 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:10:42,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:48,046 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:48,194 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:48,747 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:10:53,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:10:53,194 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:10:53,400 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:10:57,822 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:10:58,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:02,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:03,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:08,702 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:09,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:09,903 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:11:14,557 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:17,877 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:19,983 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:11:20,203 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:22,878 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:23,401 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:11:25,402 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:30,086 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:11:31,281 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:32,941 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:36,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:38,160 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:40,158 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:11:41,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:46,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:47,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:50,224 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:11:51,866 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:11:52,721 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:11:53,401 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:11:57,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:00,312 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:12:03,014 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:03,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:08,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:08,370 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:10,400 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:12:13,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:18,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:18,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:20,465 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:12:23,402 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:12:23,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:24,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:28,898 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 10:12:28,899 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 10:12:28,900 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 10:12:28,901 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 10:12:29,532 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 10:12:29,904 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:30,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:12:31,541 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:12:32,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:34,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:37,662 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:40,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:40,651 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:12:45,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:48,241 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:50,901 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:12:51,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:12:53,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:12:53,402 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:12:57,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:00,965 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:13:03,077 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:03,290 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:08,133 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:08,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:10,046 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:13:14,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:18,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:19,079 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:20,137 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:13:23,403 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:13:23,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:24,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:29,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:32,224 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:13:34,651 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:35,114 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:40,049 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:40,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:42,289 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:13:45,651 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:49,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:50,652 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:13:52,374 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:13:53,403 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:13:54,148 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:13:56,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:01,654 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:04,500 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:14:04,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:07,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:10,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:12,914 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:14,576 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:14:18,652 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:21,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:23,404 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:14:24,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:24,656 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:14:26,048 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:29,652 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:33,885 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:34,762 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:14:35,081 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:38,886 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:40,082 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:44,853 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:14:45,652 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:48,708 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:50,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:14:53,405 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:14:54,395 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:14:54,939 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:14:55,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:01,652 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:04,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:05,049 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:15:07,473 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:09,292 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:12,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:15,132 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:15:17,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:18,785 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:22,667 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:23,406 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:15:24,160 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:24,223 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:15:27,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:33,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:33,807 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:36,325 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:15:38,808 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:39,103 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:44,384 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:46,398 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:15:49,188 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:49,384 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:53,411 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:15:54,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 10:15:54,259 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-19 10:15:54,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-19 10:15:54,262 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 10:15:54,473 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-19 10:15:54,867 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:15:55,264 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:15:55,477 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:15:56,483 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:16:00,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:03,808 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:16:05,980 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:06,574 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:16:08,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:16:10,981 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:16,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:16,657 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:16:19,649 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:16:21,014 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:23,423 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-19 10:16:25,051 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:16:26,436 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:26,720 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:16:31,654 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:34,022 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:16:36,784 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:16:37,297 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:39,023 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-19 10:16:42,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 10:16:46,855 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-19 10:16:47,562 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report