diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -15363,3 +15363,5951 @@ 2023-11-22 19:11:49,738 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status 2023-11-22 19:11:49,740 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status 2023-11-22 19:11:49,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:11:53,218 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:11:53,225 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:11:55,226 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:00,227 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:03,298 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:12:03,300 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:12:03,301 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:12:03,304 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:12:04,305 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:12:04,307 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:12:04,736 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:12:04,737 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:12:05,311 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:12:05,942 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:10,944 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:16,010 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:17,468 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:12:19,737 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:12:19,737 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:12:21,940 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:23,226 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:12:27,198 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:27,574 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:12:32,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:35,397 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:12:35,398 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:12:38,030 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:39,682 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:12:43,032 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:48,034 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:49,772 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:12:50,397 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:12:50,397 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:12:53,226 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:12:53,227 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:12:58,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:01,855 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:13:03,379 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:05,397 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:13:05,398 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:13:08,566 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:11,933 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:13:14,401 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:19,415 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:20,399 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:13:20,400 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:13:20,401 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:13:22,000 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:13:23,227 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:13:25,227 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:30,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:34,075 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:13:35,397 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:13:35,398 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:13:35,607 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:40,608 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:44,150 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:13:46,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:50,398 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:13:50,399 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:13:51,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:13:53,205 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:13:53,227 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:13:56,228 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:01,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:03,278 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:14:05,398 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:14:05,398 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:14:06,644 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:11,646 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:13,342 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:14:17,079 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:21,110 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:14:21,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:14:21,113 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:14:22,271 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:23,228 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:14:23,420 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:14:28,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:33,498 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:14:33,521 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:36,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:14:36,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:14:36,112 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:14:39,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:44,826 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:45,579 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:14:49,828 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:51,824 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:14:51,825 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:14:51,826 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:14:53,229 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:14:55,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:14:55,646 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:00,232 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:05,715 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:06,120 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:06,824 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:15:06,824 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:15:12,053 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:15,795 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:17,925 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:22,927 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:22,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:15:22,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:15:22,961 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:15:23,230 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:15:25,868 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:28,201 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:33,203 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:34,843 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:15:34,844 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:15:34,845 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:15:34,848 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:15:34,942 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:15:36,952 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:37,957 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:37,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:15:37,958 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:15:39,119 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:44,121 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:48,037 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:49,121 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:53,231 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:15:54,233 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:15:54,274 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:15:54,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:15:54,277 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:15:58,106 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:15:59,573 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:04,574 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:08,180 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:16:09,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:16:09,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:16:09,275 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:16:10,538 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:15,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:20,255 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:16:21,093 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:23,231 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:16:24,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:16:24,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:16:24,279 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:16:26,512 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:30,331 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:16:32,383 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:37,384 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:39,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:16:39,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:16:39,276 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:16:40,400 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:16:42,552 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:47,554 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:49,474 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:16:52,599 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:53,232 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:16:54,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:16:54,276 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:16:58,065 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:16:59,547 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:17:03,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:08,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:09,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:17:09,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:17:09,276 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:17:09,632 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:17:13,511 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:18,512 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:21,720 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:17:23,232 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:17:24,233 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:24,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:17:24,277 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:17:29,486 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:31,783 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:17:34,650 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:39,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:17:39,277 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:17:39,277 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:17:39,703 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:41,860 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:17:44,713 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:49,714 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:17:51,947 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:17:53,233 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:17:54,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:17:54,277 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:17:55,459 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:00,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:02,010 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:18:05,771 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:09,659 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:18:09,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:18:09,661 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:18:10,824 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:12,073 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:18:15,825 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:20,826 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:23,234 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:18:24,150 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:18:24,659 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:18:24,660 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:18:25,910 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:30,912 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:34,227 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:18:36,274 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:40,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:18:40,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:18:40,261 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:18:41,383 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:44,303 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:18:46,384 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:51,386 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:18:53,234 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:18:54,389 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:18:55,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:18:55,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:18:55,259 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:18:56,504 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:01,506 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:04,272 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:19:04,273 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:19:04,274 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:19:04,277 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:19:04,460 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:19:06,474 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:19:07,285 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:07,478 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:19:10,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:19:10,260 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:19:10,261 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:19:12,517 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:16,539 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:19:18,034 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:23,035 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:23,236 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:19:25,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:19:25,260 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:19:25,261 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:19:28,665 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:19:28,880 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:33,881 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:37,740 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:19:39,529 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:40,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:19:40,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:19:40,260 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:19:45,501 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:47,817 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:19:51,037 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:53,236 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:19:56,237 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:19:56,457 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:19:56,458 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:19:56,459 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:19:59,901 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:20:01,793 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:06,795 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:09,964 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:20:11,457 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:20:11,458 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:20:11,459 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:20:12,707 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:17,709 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:20,034 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:20:23,237 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:20:23,237 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:27,506 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:20:27,507 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:20:27,509 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:20:28,783 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:30,095 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:20:33,830 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:38,832 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:40,161 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:20:42,507 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:20:42,507 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:20:42,507 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:20:44,706 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:50,424 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:52,257 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:20:53,237 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:20:56,238 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:20:58,266 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:20:58,267 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:20:58,268 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:21:01,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:02,333 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:21:06,434 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:11,768 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:12,398 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:21:13,266 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:21:13,266 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:21:13,267 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:21:17,496 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:22,464 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:21:23,140 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:23,246 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:21:28,266 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:29,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:21:29,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:21:29,433 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:21:32,543 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:21:34,091 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:39,093 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:44,374 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:44,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:21:44,431 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:21:44,626 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:21:49,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:21:53,246 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:21:54,686 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:21:55,247 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:00,249 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:00,804 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:22:00,810 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:22:00,811 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:22:04,748 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:22:06,136 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:11,138 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:14,819 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:22:15,804 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:22:15,805 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:22:15,805 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:22:17,021 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:22,022 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:23,247 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:22:24,915 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:22:27,248 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:31,277 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:22:31,279 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:22:31,280 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:22:32,469 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:32,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:22:32,667 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:22:32,668 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:22:32,670 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:22:32,975 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:22:33,980 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:22:34,985 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:22:37,673 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:42,675 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:44,055 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:22:46,278 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:22:46,279 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:22:46,280 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:22:48,438 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:53,248 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:22:53,600 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:22:56,236 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:22:58,606 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:02,849 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:23:02,850 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:23:02,851 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:23:04,077 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:06,294 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:23:09,222 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:14,224 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:16,353 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:23:17,847 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:23:17,847 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:23:20,069 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:23,250 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:23:26,027 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:26,429 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:23:31,028 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:34,009 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:23:34,010 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:23:34,012 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:23:36,209 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:36,499 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:23:41,210 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:46,567 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:23:47,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:49,010 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:23:49,010 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:23:52,195 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:53,251 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:23:57,643 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:23:58,652 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:24:02,645 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:04,591 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:24:04,593 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:24:04,594 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:24:07,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:08,726 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:24:12,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:18,385 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:18,793 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:24:19,591 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:24:19,592 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:24:23,253 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:24:24,255 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:28,868 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:24:30,139 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:35,141 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:35,324 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:24:35,326 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:24:35,328 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:24:38,938 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:24:40,623 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:45,625 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:49,019 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:24:50,324 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:24:50,325 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:24:51,565 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:53,253 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:24:57,130 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:24:59,104 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:25:02,131 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:06,132 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:25:06,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:25:06,135 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:25:07,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:08,177 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:25:12,323 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:17,324 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:20,245 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:25:21,132 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:25:21,133 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:25:23,253 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:25:23,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:28,255 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:33,256 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:34,323 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:25:36,133 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:25:36,133 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:25:38,366 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:43,367 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:46,401 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:25:48,503 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:51,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:25:51,135 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:25:53,255 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:25:54,257 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:25:56,471 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:25:59,529 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:04,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:05,296 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:26:05,305 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:26:05,305 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:26:05,308 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:26:05,528 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:26:06,133 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:26:06,133 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:26:06,533 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:26:07,538 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:26:10,341 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:15,343 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:16,596 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:26:20,455 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:21,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:26:21,134 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:26:23,260 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:26:25,643 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:26,667 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:26:30,644 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:35,646 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:36,133 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:26:36,133 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:26:36,740 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:26:41,353 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:46,995 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:48,816 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:26:51,133 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:26:51,134 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:26:52,310 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:53,258 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:26:58,144 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:26:58,885 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:27:03,145 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:06,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:27:06,135 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:27:08,517 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:08,953 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:27:13,519 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:18,730 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:19,043 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:27:21,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:27:21,135 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:27:23,257 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:27:24,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:29,108 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:27:29,731 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:34,733 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:36,135 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:27:36,137 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:27:39,201 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:27:40,024 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:45,039 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:49,285 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:27:50,370 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:51,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:27:51,135 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:27:53,258 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:27:56,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:27:59,352 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:28:01,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:06,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:07,165 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:28:07,167 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:28:07,169 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:28:10,439 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:28:12,319 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:17,321 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:20,511 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:28:22,165 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:28:22,166 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:28:22,166 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:28:22,331 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:23,259 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:28:28,260 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:30,585 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:28:33,657 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:37,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:28:37,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:28:37,535 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:28:38,789 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:40,654 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:28:43,940 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:48,942 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:50,715 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:28:52,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:28:52,533 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:28:53,259 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:28:54,260 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:28:59,262 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:02,799 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:29:04,317 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:07,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:29:07,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:29:07,537 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:29:09,751 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:12,872 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:29:15,656 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:20,657 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:22,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:29:22,534 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:29:23,260 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:29:24,951 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:29:26,262 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:31,263 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:33,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:29:33,112 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:29:33,113 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:29:33,115 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:29:34,050 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:29:35,056 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:29:36,065 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:29:37,117 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:37,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:29:37,535 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:29:42,712 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:47,171 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:29:48,131 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:52,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:29:52,534 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:29:53,262 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:29:53,267 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:29:57,243 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:29:58,852 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:03,853 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:07,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:30:07,534 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:30:09,326 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:30:09,754 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:14,756 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:19,389 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:30:19,954 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:22,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:30:22,535 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:30:23,262 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:30:25,263 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:29,464 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:30:30,407 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:35,374 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:37,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:30:37,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:30:37,536 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:30:41,222 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:41,594 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:30:46,223 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:50,665 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:30:51,894 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:30:52,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:30:52,535 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:30:53,273 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:30:57,274 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:02,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:07,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:07,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:31:07,535 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:31:12,791 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:17,792 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:22,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:31:22,537 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:31:22,538 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:31:22,793 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:23,263 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:31:25,200 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:31:28,265 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:33,266 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:35,288 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:31:37,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:31:37,535 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:31:38,791 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:43,793 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:47,383 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:31:49,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:53,265 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:31:53,457 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:31:53,459 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:31:53,461 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:31:54,639 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:31:57,458 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:31:59,777 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:04,779 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:07,534 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:32:08,457 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:32:08,457 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:32:10,718 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:15,720 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:17,609 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:32:20,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:23,266 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:32:25,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:32:25,112 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:32:25,113 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:32:26,369 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:29,694 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:32:31,689 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:36,691 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:39,764 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:32:40,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:32:40,112 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:32:40,112 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:32:42,312 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:47,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:48,860 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:32:53,177 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:32:53,267 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:32:55,112 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:32:55,112 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:32:55,113 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:32:58,285 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:00,982 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:33:04,122 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:09,124 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:10,204 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:33:10,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:33:10,207 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:33:13,064 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:33:14,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:19,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:23,267 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:33:25,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:33:25,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:33:25,206 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:33:25,321 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:33:25,417 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:30,418 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:33,706 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:33:33,707 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:33:33,709 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:33:33,713 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:33:34,404 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:33:35,411 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:33:35,716 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:36,420 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:33:40,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:33:40,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:33:40,206 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:33:41,465 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:46,853 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:47,504 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:33:51,854 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:53,268 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:33:55,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:33:55,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:33:55,207 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:33:57,600 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:33:57,633 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:34:02,601 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:07,931 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:09,738 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:34:10,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:34:10,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:34:10,206 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:34:13,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:18,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:19,800 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:34:23,269 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:34:24,269 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:25,207 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:34:25,208 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:34:25,210 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:34:29,554 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:31,892 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:34:34,555 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:39,907 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:40,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:34:40,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:34:40,206 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:34:40,964 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:34:45,453 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:50,463 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:34:51,056 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:34:53,272 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:34:55,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:34:55,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:34:55,207 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:34:56,457 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:01,292 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:35:02,069 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:07,071 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:10,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:35:10,207 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:35:10,208 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:35:11,367 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:35:12,488 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:17,489 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:23,052 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:23,271 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:35:23,478 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:35:25,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:35:25,207 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:35:25,207 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:35:28,428 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:33,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:33,552 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:35:38,501 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:40,829 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:35:40,830 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:35:40,831 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:35:43,625 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:35:44,399 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:49,400 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:53,271 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:35:55,272 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:35:55,828 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:35:55,829 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:36:01,089 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:06,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:10,829 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:36:10,829 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:36:12,021 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:17,751 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:19,824 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:36:22,752 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:23,272 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:36:26,472 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:36:26,473 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:36:26,474 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:36:28,091 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:29,890 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:36:33,093 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:38,094 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:41,472 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:36:41,473 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:36:41,474 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:36:41,964 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:36:43,722 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:48,723 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:52,035 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:36:53,272 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:36:54,273 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:36:56,473 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:36:56,474 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:36:56,475 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:36:59,626 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:03,145 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:37:05,502 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:10,504 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:11,475 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:37:11,478 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:37:15,283 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:37:16,393 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:21,394 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:23,275 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:37:26,473 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:26,475 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:37:26,475 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:37:27,380 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:37:31,723 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:35,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:37:35,091 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:37:35,092 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:37:35,095 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:37:35,439 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:37:36,444 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:37:37,097 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:37,450 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:37:41,475 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:37:41,476 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:37:42,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:47,519 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:37:48,321 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:53,276 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:37:54,278 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:37:56,475 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:37:56,476 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:37:57,591 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:37:59,643 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:04,644 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:07,665 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:38:10,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:11,476 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:38:11,477 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:38:15,731 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:19,749 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:38:20,818 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:23,276 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:38:26,277 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:26,476 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:38:26,478 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:38:29,812 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:38:31,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:36,662 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:39,889 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:38:41,476 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:38:41,477 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:38:42,631 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:47,633 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:49,953 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:38:53,277 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:38:53,278 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:38:57,412 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:38:57,413 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:38:57,414 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:38:58,613 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:00,025 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:39:03,614 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:08,615 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:12,107 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:39:12,412 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:39:12,413 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:39:12,413 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:39:13,673 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:18,675 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:22,182 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:39:23,277 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:39:24,277 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:28,750 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:39:28,752 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:39:28,753 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:39:30,019 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:32,262 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:39:35,424 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:40,425 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:42,350 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:39:43,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:39:43,763 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:39:43,763 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:39:45,951 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:50,953 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:53,278 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:39:53,435 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:39:56,279 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:39:58,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:39:58,763 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:39:58,763 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:40:01,982 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:03,513 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:40:07,493 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:12,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:13,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:40:13,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:40:13,763 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:40:15,582 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:40:17,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:22,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:23,278 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:40:25,646 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:40:28,279 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:28,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:40:28,763 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:40:28,764 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:40:33,510 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:35,716 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:40:38,517 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:43,519 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:43,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:40:43,763 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:40:43,764 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:40:45,794 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:40:49,227 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:53,280 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:40:54,283 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:40:57,881 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:40:58,763 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:40:58,763 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:40:58,764 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:40:59,955 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:04,957 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:05,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:41:05,277 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:41:05,277 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:41:05,280 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:41:05,944 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:41:06,948 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:41:07,954 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:41:10,282 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:14,586 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:41:14,588 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:41:14,589 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:41:15,826 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:18,012 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:41:20,844 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:23,293 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:41:26,034 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:28,084 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:41:29,587 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:41:29,588 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:41:29,589 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:41:31,767 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:37,360 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:38,162 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:41:42,361 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:45,212 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:41:45,215 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:41:45,217 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:41:47,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:48,233 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:41:52,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:41:53,281 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:41:58,282 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:01,121 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:42:01,122 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:42:01,123 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:42:03,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:04,348 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:42:08,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:13,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:14,426 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:42:16,121 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:42:16,121 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:42:16,122 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:42:19,298 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:23,282 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:42:24,354 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:26,518 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:42:29,355 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:32,684 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:42:32,685 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:42:32,686 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:42:35,012 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:36,583 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:42:40,013 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:45,249 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:45,648 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:42:47,685 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:42:47,685 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:42:50,912 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:42:53,284 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:42:55,722 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:42:56,424 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:01,425 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:03,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:43:03,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:43:03,982 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:43:05,792 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:43:07,162 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:12,164 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:17,440 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:17,882 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:43:18,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:43:18,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:43:18,979 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:43:23,233 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:23,285 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:43:27,946 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:43:28,753 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:33,755 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:34,583 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:43:34,585 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:43:34,587 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:43:38,015 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:43:38,955 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:43,957 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:48,089 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:43:49,119 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:49,584 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:43:49,585 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:43:49,585 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:43:53,287 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:43:54,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:43:58,179 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:43:59,540 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:04,541 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:05,948 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:44:05,949 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:44:05,950 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:44:08,253 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:44:10,139 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:15,140 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:20,342 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:44:20,368 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:20,948 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:44:20,949 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:44:20,950 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:44:23,287 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:44:26,288 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:30,415 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:44:31,718 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:36,551 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:44:36,553 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:44:36,554 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:44:36,792 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:37,935 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:44:37,936 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:44:37,937 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:44:37,939 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:44:38,470 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:44:40,481 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:44:41,941 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:46,942 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:50,548 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:44:51,552 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:44:51,554 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:44:51,555 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:44:52,726 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:44:53,287 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:44:58,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:00,618 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:45:03,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:07,482 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:45:07,483 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:45:07,485 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:45:08,657 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:10,709 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:45:13,657 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:18,659 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:20,792 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:45:22,482 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:45:22,483 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:45:22,484 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:45:23,288 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:45:24,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:29,761 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:29,852 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:45:34,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:37,769 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:45:37,771 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:45:37,772 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:45:39,938 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:45:39,984 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:44,985 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:50,548 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:45:52,016 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:45:52,769 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:45:52,770 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:45:52,771 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:45:53,289 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:45:56,290 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:01,459 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:02,075 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:46:06,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:08,307 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:46:08,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:46:08,309 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:46:11,766 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:12,145 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:46:14,159 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:46:17,706 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:22,219 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:46:23,289 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:46:23,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:23,307 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:46:23,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:46:23,308 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:46:28,552 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:32,289 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:46:33,690 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:38,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:46:38,310 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:46:38,312 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:46:39,564 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:42,364 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:46:45,247 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:50,248 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:46:53,289 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:46:53,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:46:53,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:46:53,309 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:46:54,446 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:46:55,556 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:00,557 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:04,512 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:47:05,687 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:08,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:47:08,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:47:08,310 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:47:10,728 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:14,581 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:47:15,890 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:20,892 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:23,290 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:47:23,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:47:23,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:47:23,310 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:47:24,648 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:47:26,557 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:31,559 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:34,721 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:47:37,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:38,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:47:38,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:47:38,310 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:47:42,547 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:44,798 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:47:47,803 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:52,804 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:47:53,291 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:47:53,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:47:53,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:47:53,309 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:47:54,871 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:47:58,490 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:03,317 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:48:03,339 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:48:03,345 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:48:03,347 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:48:03,934 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:48:04,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:04,940 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:48:08,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:48:08,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:48:08,310 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:48:09,546 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:14,805 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:17,019 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:48:19,806 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:23,291 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:48:23,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:48:23,310 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:48:23,311 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:48:24,919 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:26,107 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:48:29,920 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:35,301 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:36,212 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:48:38,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:48:38,310 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:48:38,311 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:48:40,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:46,054 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:46,337 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:48:51,055 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:53,293 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:48:53,883 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:48:53,883 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:48:53,885 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:48:56,256 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:48:56,404 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:49:01,257 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:06,337 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:06,475 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:49:08,883 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:49:08,884 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:49:08,885 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:49:12,055 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:16,554 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:49:17,512 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:22,514 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:23,294 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:49:24,418 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:49:24,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:49:24,420 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:49:27,770 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:28,642 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:49:32,771 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:38,609 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:38,717 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:49:39,425 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:49:39,426 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:49:43,653 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:48,782 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:49:49,459 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:49:53,294 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:49:54,425 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:49:54,426 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:49:54,650 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:00,093 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:00,869 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:50:05,094 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:09,426 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:50:09,427 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:50:10,674 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:10,938 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:50:15,676 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:20,786 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:21,007 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:50:23,295 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:50:24,426 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:50:24,427 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:50:26,684 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:31,083 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:50:32,382 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:37,383 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:39,426 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:50:39,427 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:50:43,173 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:50:43,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:48,310 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:53,242 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:50:53,295 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:50:54,296 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:50:54,427 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:50:54,428 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:50:59,671 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:02,309 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:51:05,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:09,426 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:51:09,427 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:51:10,667 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:15,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:20,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:23,309 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:51:24,583 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:51:24,585 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:51:24,586 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:51:25,795 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:28,444 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:51:31,145 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:36,147 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:38,527 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:51:39,582 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:51:39,583 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:51:41,797 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:46,799 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:48,212 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:51:48,215 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:51:48,215 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:51:48,217 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:51:48,599 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:51:49,604 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:51:50,610 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:51:52,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:51:53,297 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:51:54,583 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:51:54,584 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:51:57,750 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:00,677 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:52:02,863 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:07,865 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:09,583 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:52:09,583 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:52:10,745 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:52:13,767 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:18,769 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:20,819 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:52:23,298 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:52:24,299 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:24,583 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:52:24,584 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:52:29,759 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:32,904 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:52:34,945 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:39,747 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:52:39,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:52:39,750 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:52:39,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:43,099 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:52:45,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:50,208 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:52:53,299 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:52:54,747 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:52:54,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:52:54,749 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:52:55,199 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:52:55,933 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:00,935 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:05,296 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:53:06,119 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:09,747 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:53:09,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:53:09,749 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:53:11,930 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:15,375 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:53:17,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:22,672 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:23,301 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:53:24,747 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:53:24,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:53:24,749 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:53:27,457 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:53:27,956 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:32,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:36,527 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:53:38,557 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:39,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:53:39,749 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:53:39,749 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:53:43,936 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:46,606 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:53:48,990 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:53,302 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:53:54,305 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:53:54,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:53:54,750 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:53:54,750 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:53:56,675 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:53:59,344 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:04,346 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:08,758 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:54:09,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:54:09,749 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:54:09,750 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:09,750 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:54:14,955 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:18,831 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:54:20,502 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:23,304 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:54:25,338 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:54:25,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:54:25,341 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:54:25,591 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:28,909 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:54:31,580 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:36,582 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:38,990 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:54:40,338 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:54:40,339 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:54:42,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:47,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:49,063 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:54:52,787 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:54:53,303 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:54:55,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:54:55,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:54:55,342 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:54:58,341 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:01,137 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:55:03,342 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:08,343 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:10,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:55:10,340 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:55:11,205 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:55:13,566 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:18,567 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:19,517 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:55:19,518 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:55:19,519 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:55:19,522 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:55:20,286 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:55:21,295 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:55:23,305 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:55:24,306 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:25,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:55:25,340 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:55:29,545 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:31,375 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:55:35,192 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:40,193 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:40,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:55:40,341 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:55:43,465 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:55:45,650 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:50,651 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:55:53,305 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:55:53,537 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:55:55,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:55:55,340 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:55:56,574 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:01,576 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:03,622 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:56:07,176 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:11,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:56:11,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:56:11,533 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:56:12,840 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:14,730 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:56:17,896 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:22,897 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:23,306 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:56:24,798 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:56:26,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:56:26,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:56:26,531 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:56:28,761 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:34,759 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:34,862 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:56:39,772 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:42,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:56:42,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:56:42,667 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:56:44,970 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:56:45,152 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:50,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:53,309 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:56:55,312 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:56:57,061 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:56:57,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:56:57,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:56:57,666 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:57:00,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:05,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:09,141 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:57:11,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:12,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:57:12,667 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:57:12,668 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:57:16,815 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:19,210 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:57:21,816 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:23,308 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:57:27,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:27,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:57:27,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:57:27,666 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:57:29,288 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:57:32,894 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:37,896 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:41,381 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:57:42,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:57:42,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:57:42,666 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:57:42,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:47,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:52,972 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:57:53,310 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:57:57,480 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:57:57,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:57:57,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:57:57,667 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:57:58,917 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:03,919 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:07,586 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:58:08,955 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:12,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:58:12,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:58:12,667 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:58:14,924 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:19,663 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:58:20,603 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:23,310 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:58:26,312 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:27,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:58:27,667 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:58:27,668 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:58:29,729 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:58:31,884 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:36,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:39,803 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:58:41,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:42,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:58:42,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:58:42,667 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:58:47,930 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:51,887 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:58:53,310 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:58:53,311 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:57,939 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:58:57,941 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:58:57,943 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:58:59,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:58:59,285 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 19:58:59,288 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 19:58:59,288 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 19:58:59,290 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 19:58:59,977 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 19:59:00,983 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:59:01,989 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:59:04,292 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:09,294 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:11,054 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:59:12,939 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:59:12,941 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:59:12,941 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:59:15,179 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:20,745 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:21,126 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:59:23,311 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:59:26,312 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:29,064 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 19:59:29,065 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:59:29,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:59:31,207 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:59:31,444 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:36,450 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:42,051 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:43,305 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:59:44,065 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:59:44,066 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 19:59:47,280 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:52,514 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:53,312 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 19:59:53,389 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 19:59:58,313 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 19:59:59,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 19:59:59,068 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:00:03,462 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:00:03,928 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:08,929 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:13,539 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:00:14,065 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:14,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:00:14,066 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:00:19,246 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:23,313 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:00:24,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:25,641 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:00:29,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:00:29,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:00:30,271 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:35,720 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:00:35,872 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:40,873 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:44,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:00:44,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:00:46,691 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:47,807 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:00:51,692 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:53,315 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:00:57,481 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:00:57,885 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:00:59,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:00:59,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:01:03,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:09,045 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:09,976 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:01:14,046 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:14,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:01:14,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:01:19,680 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:20,051 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:01:23,356 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:01:25,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:29,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:01:29,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:01:30,140 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:01:31,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:36,291 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:39,204 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:01:41,571 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:44,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:01:44,068 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:01:47,343 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:51,280 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:01:52,925 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:53,315 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:01:58,317 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:01:59,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:01:59,068 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:02:01,390 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:02:03,873 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:08,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:11,611 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:02:14,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:14,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:02:14,068 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:02:19,294 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:21,745 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:02:23,316 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:02:24,317 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:29,319 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:29,904 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:02:29,907 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:02:29,908 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:02:31,645 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:02:31,647 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:02:31,647 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:02:31,650 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:02:31,825 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:02:33,840 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:02:34,652 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:39,654 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:43,913 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:02:44,719 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:44,905 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:02:44,906 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:02:44,907 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:02:50,115 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:53,316 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:02:55,382 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:02:56,004 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:02:59,905 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:02:59,905 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:02:59,906 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:03:01,127 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:06,127 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:03:06,813 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:11,814 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:14,905 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:03:14,906 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:03:14,908 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:03:17,696 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:18,196 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:03:22,698 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:23,320 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:03:27,810 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:28,264 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:03:29,905 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:03:29,906 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:03:29,906 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:03:33,131 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:38,250 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:38,355 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:03:43,252 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:45,257 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:03:45,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:03:45,260 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:03:47,441 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:03:48,621 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:53,320 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:03:54,322 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:03:59,538 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:03:59,571 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:00,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:04:00,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:04:00,259 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:04:04,573 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:09,608 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:04:10,218 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:15,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:15,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:04:15,260 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:04:15,262 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:04:19,682 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:04:20,656 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:23,321 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:04:26,323 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:29,778 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:04:30,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:04:30,259 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:04:31,429 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:36,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:41,869 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:04:42,331 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:45,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:04:45,260 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:04:47,439 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:51,943 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:04:52,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:04:53,322 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:04:58,324 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:00,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:05:00,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:05:00,260 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:05:02,004 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:05:03,469 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:08,471 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:12,071 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:05:14,020 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:15,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:05:15,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:05:15,260 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:05:19,439 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:23,322 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:05:24,158 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:05:25,323 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:30,325 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:31,071 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:05:31,071 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:05:31,073 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:05:34,237 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:05:35,503 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:40,506 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:44,316 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:05:45,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:46,071 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:05:46,072 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:05:46,073 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:05:51,232 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:05:53,323 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:05:54,388 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:05:56,324 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:01,326 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:02,189 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:06:02,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:06:02,192 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:06:03,589 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:06:03,591 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:06:03,592 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:06:03,595 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:06:04,494 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:06:06,505 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:06:06,598 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:11,599 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:16,592 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:06:16,704 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:17,189 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:06:17,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:06:17,191 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:06:22,371 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:23,324 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:06:25,656 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:06:28,287 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:32,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:06:32,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:06:32,191 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:06:33,424 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:37,765 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:06:39,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:44,366 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:47,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:06:47,191 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:06:47,192 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:06:49,576 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:49,839 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:06:53,325 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:06:55,327 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:06:59,911 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:07:00,677 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:02,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:07:02,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:07:02,191 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:07:06,387 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:09,981 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:07:11,940 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:16,941 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:18,378 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:07:18,380 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:07:18,381 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:07:22,055 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:07:22,637 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:23,326 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:07:28,327 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:32,156 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:07:33,379 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:07:33,379 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:33,380 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:07:38,565 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:42,231 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:07:43,712 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:48,713 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:49,849 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:07:49,851 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:07:49,852 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:07:52,290 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:07:53,326 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:07:54,327 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:07:59,329 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:04,428 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:08:04,861 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:08:04,863 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:04,864 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:08:04,864 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:08:10,119 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:14,503 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:08:15,833 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:19,862 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:08:19,862 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:08:19,863 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:08:21,107 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:23,327 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:08:26,592 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:08:26,695 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:31,697 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:34,862 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:08:34,868 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:08:34,869 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:08:36,665 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:08:37,204 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:42,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:46,749 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:08:47,443 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:49,862 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:08:49,862 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:08:49,863 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:08:53,049 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:08:53,329 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:08:55,824 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:08:58,490 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:03,492 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:04,863 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:09:04,864 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:09:04,866 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:09:07,914 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:09:08,878 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:13,899 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:17,979 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:09:19,435 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:19,862 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:09:19,863 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:09:19,863 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:09:23,330 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:09:25,332 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:28,056 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:09:30,683 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:35,685 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:36,187 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:09:36,189 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:09:36,190 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:09:37,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:09:37,877 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:09:37,879 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:09:37,884 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:09:38,143 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:09:40,155 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:09:40,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:45,887 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:50,238 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:09:51,187 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:09:51,187 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:09:51,188 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:09:53,330 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:09:56,331 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:00,329 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:10:01,562 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:06,563 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:07,895 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:10:07,896 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:10:07,897 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:10:11,701 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:12,444 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:10:16,702 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:22,114 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:22,525 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:10:22,894 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:10:22,895 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:10:23,331 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:10:27,332 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:32,786 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:34,614 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:10:37,787 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:37,895 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:10:37,896 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:10:37,896 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:10:43,478 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:44,695 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:10:48,479 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:52,895 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:10:52,896 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:10:53,331 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:10:54,332 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:10:54,773 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:10:59,333 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:04,624 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:04,853 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:11:07,895 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:11:07,896 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:11:10,149 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:15,249 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:16,942 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:11:20,250 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:23,352 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:11:23,949 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:11:23,951 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:11:23,952 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:11:25,298 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:27,018 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:11:30,305 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:35,307 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:36,080 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:11:38,950 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:11:38,951 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:11:38,952 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:11:41,175 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:46,145 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:11:46,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:51,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:53,335 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:11:55,172 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:11:55,173 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:11:55,174 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:11:57,627 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:11:58,230 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:12:02,628 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:08,300 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:12:08,469 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:10,173 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:12:10,173 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:12:10,174 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:12:14,335 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:19,838 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:20,396 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:12:23,335 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:12:25,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:12:25,177 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:12:25,179 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:25,180 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:12:30,380 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:30,494 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:12:35,381 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:40,173 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:12:40,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:12:40,174 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:12:40,558 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:12:41,373 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:46,375 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:52,252 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:12:52,636 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:12:53,335 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:12:55,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:12:55,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:12:55,175 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:12:57,363 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:02,733 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:13:02,812 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:07,813 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:10,672 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:13:10,673 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:13:10,674 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:13:12,629 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:13:12,631 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:13:12,632 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:13:12,636 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:13:12,831 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:13:13,638 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:14,842 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:13:18,639 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:23,336 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:13:24,337 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:26,498 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:13:26,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:13:26,501 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:13:29,937 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:30,983 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:13:34,938 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:40,381 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:41,069 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:13:41,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:13:41,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:13:41,500 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:13:45,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:51,075 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:51,148 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:13:53,336 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:13:56,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:13:56,501 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:13:56,513 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:13:56,515 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:14:01,232 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:14:01,853 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:06,854 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:11,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:14:11,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:14:11,501 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:14:12,305 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:14:12,707 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:17,709 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:22,389 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:14:23,094 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:23,337 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:14:26,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:14:26,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:14:26,501 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:14:28,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:32,475 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:14:33,738 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:38,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:41,501 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:14:41,503 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:14:41,505 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:14:43,903 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:44,582 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:14:48,904 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:53,338 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:14:54,559 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:14:54,645 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:14:56,501 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:14:56,502 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:14:59,682 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:04,725 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:15:04,794 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:09,795 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:12,819 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:15:12,820 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:15:12,821 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:15:15,557 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:16,812 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:15:20,558 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:23,340 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:15:25,834 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:26,930 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:15:27,819 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:15:27,819 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:15:27,820 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:15:31,031 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:36,033 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:39,017 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:15:41,598 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:42,819 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:15:42,830 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:15:46,730 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:49,150 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:15:51,731 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:53,340 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:15:57,341 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:15:57,819 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:15:57,819 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:15:59,223 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:16:03,068 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:08,070 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:11,358 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:16:12,819 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:16:12,820 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:16:13,985 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:18,987 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:21,440 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:16:23,354 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:16:24,354 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:29,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:16:29,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:16:29,101 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:16:30,273 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:32,557 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:16:35,477 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:40,479 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:42,646 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:16:44,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:16:44,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:16:44,099 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:16:46,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:51,291 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:51,628 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:16:51,629 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:16:51,630 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:16:51,633 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:16:51,715 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:16:52,723 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:16:53,341 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:16:56,343 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:16:59,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:16:59,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:16:59,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:17:01,354 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:04,806 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:17:06,770 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:11,772 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:14,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:17:14,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:17:14,099 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:17:14,879 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:17:17,323 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:22,324 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:23,342 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:17:26,962 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:17:27,686 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:29,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:17:29,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:17:29,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:17:33,351 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:37,045 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:17:39,038 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:44,039 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:44,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:17:44,231 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:17:44,233 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:17:47,126 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:17:49,086 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:53,342 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:17:54,344 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:17:59,201 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:17:59,252 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:17:59,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:17:59,254 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:17:59,470 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:04,471 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:09,291 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:18:10,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:14,252 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:18:14,252 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:18:15,477 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:20,764 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:21,375 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:18:23,343 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:18:26,344 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:29,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:18:29,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:18:29,254 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:18:31,472 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:18:31,529 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:36,546 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:41,595 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:18:42,079 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:44,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:18:44,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:18:44,254 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:18:47,482 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:52,680 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:18:53,097 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:18:53,343 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:18:58,345 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:00,491 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:19:00,492 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:19:00,494 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:19:02,760 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:19:03,837 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:08,838 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:12,840 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:19:14,129 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:15,491 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:19:15,492 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:19:19,683 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:23,344 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:19:24,916 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:19:25,345 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:30,346 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:32,000 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:19:32,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:19:32,003 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:19:34,994 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:19:35,482 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:40,483 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:45,603 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:47,000 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:19:47,000 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:19:47,001 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:19:47,094 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:19:51,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:19:53,346 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:19:57,170 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:19:57,257 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:02,000 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:20:02,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:20:02,001 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:20:03,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:07,297 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:20:08,450 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:13,451 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:17,000 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:20:17,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:20:17,001 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:20:17,377 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:20:19,155 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:23,346 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:20:24,347 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:26,946 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:20:26,947 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:20:26,948 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:20:26,951 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:20:27,450 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:20:29,462 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:20:29,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:32,000 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:20:32,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:20:32,001 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:20:35,206 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:39,549 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:20:40,626 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:45,627 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:47,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:20:47,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:20:47,005 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:20:49,632 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:20:51,434 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:20:53,347 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:20:57,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:01,731 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:21:02,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:21:02,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:21:02,002 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:21:03,228 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:08,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:12,911 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:21:13,860 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:17,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:21:17,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:21:17,002 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:21:19,252 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:22,972 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:21:23,347 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:21:24,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:29,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:32,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:21:32,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:21:32,003 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:21:33,070 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:21:34,479 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:39,480 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:44,855 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:45,155 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:21:47,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:21:47,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:21:47,004 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:21:50,242 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:21:53,348 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:21:55,231 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:21:55,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:00,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:02,977 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:22:02,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:22:02,979 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:22:05,314 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:22:06,422 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:11,444 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:16,845 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:17,428 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:22:17,977 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:22:17,977 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:22:17,978 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:22:22,210 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:23,350 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:22:27,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:27,506 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:22:32,420 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:32,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:22:32,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:22:32,981 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:22:37,580 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:22:38,233 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:43,234 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:47,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:22:47,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:22:47,980 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:22:48,884 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:51,724 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:22:53,350 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:22:54,351 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:22:59,353 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:02,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:23:02,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:23:02,979 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:23:03,843 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:23:05,201 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:10,203 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:15,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:15,941 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:23:17,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:23:17,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:23:17,980 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:23:21,235 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:23,351 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:23:27,198 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:28,014 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:23:32,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:32,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:23:32,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:23:32,981 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:23:37,095 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:23:37,610 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:42,611 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:47,612 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:47,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:23:47,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:23:47,980 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:23:49,179 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:23:53,148 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:23:53,351 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:23:58,353 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:01,289 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:24:02,978 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:24:02,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:24:02,979 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:24:04,167 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:09,169 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:10,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:24:10,309 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:24:10,310 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:24:10,313 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:24:10,371 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:24:11,379 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:24:14,315 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:19,316 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:19,659 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:24:19,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:24:19,663 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:24:23,352 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:24:23,473 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:24:24,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:29,366 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:33,554 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:24:34,512 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:34,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:24:34,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:24:34,661 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:24:39,848 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:45,644 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:24:45,769 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:49,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:24:49,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:24:49,662 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:24:50,862 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:24:53,354 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:24:55,714 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:24:56,362 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:01,363 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:04,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:25:04,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:25:04,663 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:25:07,198 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:07,796 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:25:12,200 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:17,472 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:17,882 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:25:19,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:25:19,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:25:19,662 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:25:22,911 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:23,356 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:25:27,948 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:25:28,586 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:33,597 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:34,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:25:34,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:25:34,664 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:25:38,028 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:25:38,946 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:43,947 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:48,090 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:25:49,633 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:25:49,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:25:49,662 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:25:49,662 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:25:53,356 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:25:55,358 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:00,175 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:26:01,208 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:04,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:26:04,662 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:26:04,662 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:26:06,847 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:10,257 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:26:12,660 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:17,693 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:19,662 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:26:19,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:26:19,663 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:26:21,330 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:26:22,893 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:23,371 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:26:28,375 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:31,406 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:26:33,693 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:34,661 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:26:34,662 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:26:34,663 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:26:38,929 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:41,532 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:26:43,933 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:48,934 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:50,338 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:26:50,341 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:26:50,342 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:26:53,359 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:26:53,646 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:26:54,360 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:26:59,361 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:03,718 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:27:04,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:05,338 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:27:05,339 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:27:09,606 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:15,246 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:15,800 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:27:20,248 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:20,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:27:20,342 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:27:23,360 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:27:25,680 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:25,864 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:27:30,681 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:35,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:27:35,340 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:27:35,933 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:27:36,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:41,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:45,183 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:27:45,185 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:27:45,186 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:27:45,189 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:27:46,012 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:27:46,012 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:27:47,192 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:48,022 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:27:50,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:27:50,339 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:27:52,521 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:53,361 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:27:57,833 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:27:58,098 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:28:02,834 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:05,872 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:28:05,873 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:28:05,874 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:28:08,217 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:08,261 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:28:13,226 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:19,073 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:20,351 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:28:20,873 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:28:20,875 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:28:23,362 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:28:24,363 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:30,202 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:30,416 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:28:35,204 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:37,346 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:28:37,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:28:37,348 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:28:39,485 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:28:40,724 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:45,725 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:50,897 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:28:51,595 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:28:52,346 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:28:52,347 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:28:52,348 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:28:53,362 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:28:56,364 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:01,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:01,693 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:29:06,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:07,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:29:07,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:29:07,350 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:29:11,777 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:29:11,792 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:16,793 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:21,794 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:22,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:29:22,350 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:29:23,364 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:29:27,587 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:27,884 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:29:32,590 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:37,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:29:37,349 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:29:37,971 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:29:38,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:43,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:48,051 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:29:48,864 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:52,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:29:52,349 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:29:53,364 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:29:54,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:29:59,404 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:00,134 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:30:04,406 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:07,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:30:07,350 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:30:09,867 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:10,202 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:30:14,869 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:20,273 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:30:20,542 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:22,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:30:22,349 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:30:23,364 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:30:26,366 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:31,817 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:32,365 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:30:36,835 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:37,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:30:37,349 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:30:42,438 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:30:42,443 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:47,444 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:52,361 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:30:52,361 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:30:52,362 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:30:52,575 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:30:53,365 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:30:54,517 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:30:58,367 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:03,468 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:04,589 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:31:07,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:31:07,357 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:31:08,572 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:13,573 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:16,672 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:31:18,593 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:22,559 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:31:22,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:31:22,566 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:31:23,366 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:31:23,835 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:23,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:31:23,877 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:31:23,878 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:31:23,884 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:31:24,759 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:31:25,764 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:31:28,889 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:33,890 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:35,836 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:31:37,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:31:37,570 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:31:37,606 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:31:39,831 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:44,832 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:45,912 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:31:50,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:52,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:31:52,562 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:31:52,564 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:31:53,368 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:31:55,371 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:31:58,007 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:32:01,256 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:06,257 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:07,560 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:32:07,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:32:07,561 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:32:08,079 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:32:11,801 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:16,803 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:18,161 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:32:22,207 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:22,560 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:32:22,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:32:22,562 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:32:23,368 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:32:27,370 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:30,239 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:32:32,890 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:37,562 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:32:37,562 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:32:37,564 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:32:38,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:40,326 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:32:43,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:48,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:50,405 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:32:52,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:32:52,562 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:32:52,562 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:32:53,368 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:32:54,370 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:32:59,371 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:02,484 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:33:04,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:07,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:33:07,562 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:33:07,562 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:33:09,751 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:14,565 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:33:14,823 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:19,824 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:22,566 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:33:22,566 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:33:22,567 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:33:23,370 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:33:24,647 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:33:25,371 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:30,372 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:34,728 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:33:36,046 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:37,567 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:33:37,567 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:33:37,568 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:33:41,813 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:46,801 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:33:47,746 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:52,747 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:33:52,950 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:33:52,951 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:33:52,953 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:33:53,371 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:33:56,883 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:33:58,559 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:03,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:05,947 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:34:07,950 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:34:07,951 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:34:09,173 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:14,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:16,023 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:34:19,773 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:22,952 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:34:22,953 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:34:22,954 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:34:23,373 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:34:25,376 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:28,109 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:34:31,282 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:36,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:37,952 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:34:37,953 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:34:38,181 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:34:42,178 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:47,180 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:50,264 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:34:52,904 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:52,951 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:34:52,957 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:34:53,374 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:34:58,205 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:34:58,245 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:34:58,246 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:34:58,247 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:34:58,248 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:34:58,326 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:35:00,336 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:35:03,250 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:07,953 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:35:07,955 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:35:09,211 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:10,445 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:35:14,569 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:19,578 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:22,562 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:35:22,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:35:22,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:35:22,959 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:35:23,374 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:35:25,375 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:30,377 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:32,655 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:35:35,944 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:37,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:35:37,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:35:37,962 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:35:41,054 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:42,825 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:35:46,055 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:51,057 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:35:52,903 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:35:52,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:35:52,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:35:52,961 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:35:53,375 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:35:56,376 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:01,378 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:04,983 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:36:07,157 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:07,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:36:07,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:36:07,960 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:36:12,215 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:17,089 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:36:17,513 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:22,515 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:23,377 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:36:23,390 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:36:23,391 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:36:23,393 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:36:26,146 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:36:27,674 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:32,676 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:36,225 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:36:38,267 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:38,391 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:36:38,391 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:36:43,605 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:46,372 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:36:48,682 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:53,378 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:36:53,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:36:53,393 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:36:53,395 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:36:54,607 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:36:58,458 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:36:59,923 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:04,924 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:08,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:37:08,392 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:37:08,523 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:37:10,570 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:15,572 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:18,604 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:37:21,344 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:23,378 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:37:23,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:37:23,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:37:23,393 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:37:26,575 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:28,686 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:37:32,453 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:37,455 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:38,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:37:38,392 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:37:40,791 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:37:42,617 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:47,619 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:50,882 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:37:52,696 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:37:53,379 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:37:53,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:37:53,393 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:37:58,601 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:00,950 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:38:03,716 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:08,718 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:09,028 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:38:09,030 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:38:09,032 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:38:13,044 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:38:14,382 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:19,384 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:23,106 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:38:23,379 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:38:24,028 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:38:24,029 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:38:24,029 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:38:25,232 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:30,233 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:31,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:38:31,366 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:38:31,367 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:38:31,370 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:38:32,178 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:38:32,178 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:38:33,183 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:38:35,372 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:40,368 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:38:40,369 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:38:40,370 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:38:40,608 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:43,259 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:38:46,568 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:51,570 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:38:53,327 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:38:53,380 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:38:55,368 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:38:55,369 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:38:55,369 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:38:56,586 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:01,588 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:04,399 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:39:07,203 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:11,878 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:39:11,879 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:39:11,880 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:39:13,119 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:14,461 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:39:18,194 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:23,196 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:23,381 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:39:24,542 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:39:26,879 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:39:26,879 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:39:26,880 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:39:29,129 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:34,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:34,628 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:39:39,191 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:41,881 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:39:41,883 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:39:41,884 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:39:45,176 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:46,726 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:39:50,764 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:53,382 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:39:56,387 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:39:56,880 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:39:56,880 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:39:58,837 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:40:02,140 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:07,992 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:08,921 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:40:11,880 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:40:11,881 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:40:13,047 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:18,048 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:23,049 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:23,382 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:40:26,880 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:40:26,881 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:40:28,075 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:33,077 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:37,366 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:40:38,676 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:41,880 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:40:41,881 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:40:44,093 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:47,462 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:40:49,369 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:53,383 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:40:54,384 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:40:58,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:40:58,135 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:40:58,136 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:40:59,654 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:00,565 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:41:04,655 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:09,657 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:12,662 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:41:13,135 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:41:13,135 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:41:13,135 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:41:15,381 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:20,383 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:22,735 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:41:23,384 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:41:25,386 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:28,135 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:41:28,135 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:41:31,308 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:34,835 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:41:36,998 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:41,999 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:43,697 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:41:43,697 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:41:43,699 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:41:46,917 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:41:47,592 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:52,594 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:53,403 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:41:58,186 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:41:58,697 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:41:58,697 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:41:58,698 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:41:59,008 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:42:03,879 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:09,079 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:42:09,409 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:13,697 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:42:13,698 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:42:13,699 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:42:14,948 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:20,524 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:21,163 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:42:23,386 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:42:26,387 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:28,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:42:28,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:42:28,981 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:42:30,521 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:42:30,522 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:42:30,523 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:42:30,526 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:42:31,236 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:42:31,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:32,246 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:42:33,251 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:42:36,529 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:41,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:43,325 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:42:43,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:42:43,980 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:42:47,187 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:52,189 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:53,387 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:42:55,415 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:42:57,773 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:42:58,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:42:58,980 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:43:03,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:05,498 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:43:08,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:13,434 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:13,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:43:13,981 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:43:13,981 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:43:17,589 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:43:19,150 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:23,389 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:43:24,391 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:27,664 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:43:28,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:43:28,980 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:43:30,135 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:35,136 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:38,762 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:43:40,761 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:43,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:43:43,980 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:43:46,149 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:48,831 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:43:51,271 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:53,389 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:43:56,390 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:43:58,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:43:58,981 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:43:58,985 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:44:00,914 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:44:02,250 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:07,252 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:10,976 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:44:12,545 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:13,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:44:13,981 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:44:18,179 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:21,052 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:44:23,269 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:23,390 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:44:28,391 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:29,498 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:44:29,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:44:29,501 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:44:33,134 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:44:33,912 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:38,913 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:43,222 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:44:44,498 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:44:44,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:44:44,499 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:44:44,658 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:49,659 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:53,294 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:44:53,390 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:44:55,395 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:44:59,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:44:59,502 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:44:59,503 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:45:00,691 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:05,395 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:45:05,818 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:10,819 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:14,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:45:14,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:45:14,500 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:45:15,481 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:45:16,680 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:21,688 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:23,392 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:45:27,519 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:27,558 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:45:29,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:45:29,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:45:29,501 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:45:32,714 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:37,632 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:45:38,143 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:43,144 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:44,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:45:44,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:45:44,501 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:45:48,751 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:53,392 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:45:53,743 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:45:54,393 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:45:59,395 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:00,597 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:46:00,615 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:46:00,624 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:46:03,823 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:46:04,997 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:09,998 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:12,870 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:46:12,872 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:46:12,872 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:46:12,875 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:46:12,899 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:46:14,912 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:46:15,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:46:15,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:15,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:46:15,665 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:46:15,915 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:46:20,882 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:23,393 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:46:24,977 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:46:26,682 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:30,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:46:30,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:46:30,665 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:46:31,861 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:36,895 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:37,176 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:46:41,896 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:45,674 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:46:45,676 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:46:47,251 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:46:47,843 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:52,845 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:46:53,397 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:46:57,318 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:46:58,565 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:00,673 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:47:00,674 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:47:03,920 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:07,470 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:47:09,154 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:14,155 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:15,674 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:47:15,676 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:47:19,305 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:19,567 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:47:23,394 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:47:24,395 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:29,552 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:29,655 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:47:30,674 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:47:30,675 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:47:34,848 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:39,760 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:47:39,915 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:44,916 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:45,674 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:47:45,675 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:47:50,790 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:47:51,839 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:47:53,395 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:47:56,396 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:00,686 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:48:00,687 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:48:01,906 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:48:01,944 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:06,945 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:11,974 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:48:12,624 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:15,686 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:48:15,687 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:48:17,933 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:23,290 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:23,396 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:48:24,095 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:48:28,398 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:30,686 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:48:30,687 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:48:33,883 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:35,185 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:48:39,624 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:44,626 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:45,687 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:48:45,687 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:48:47,271 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:48:49,924 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:53,398 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:48:55,326 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:48:57,346 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:49:00,327 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:00,687 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:49:00,688 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:49:05,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:07,488 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:49:10,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:15,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:15,687 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:49:15,688 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:49:17,551 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:49:21,078 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:23,400 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:49:26,402 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:27,640 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:49:30,687 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:49:30,688 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:49:31,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:36,888 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:39,725 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:49:42,328 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:46,060 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:49:46,061 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:49:46,062 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:49:47,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:47,434 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:49:47,435 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:49:47,436 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:49:47,439 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:49:47,786 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:49:48,790 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:49:49,796 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:49:52,443 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:53,399 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:49:57,814 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:49:59,864 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:50:01,060 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:50:01,061 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:50:01,061 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:50:03,313 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:09,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:09,952 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:50:14,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:17,039 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:50:17,041 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:50:17,042 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:50:19,899 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:20,021 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:50:23,400 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:50:25,401 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:30,552 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:32,093 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:50:32,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:50:32,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:50:32,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:50:36,249 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:41,889 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:42,167 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:50:46,890 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:47,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:50:47,102 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:50:47,103 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:50:52,053 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:50:52,239 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:50:53,400 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:50:57,401 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:02,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:51:02,099 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:51:03,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:04,328 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:51:08,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:13,393 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:51:14,346 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:17,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:51:17,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:51:20,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:23,400 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:51:23,458 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:51:25,401 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:30,403 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:32,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:51:32,102 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:51:32,103 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:51:35,535 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:51:36,362 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:41,363 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:46,364 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:47,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:51:47,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:51:52,277 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:51:53,403 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:51:55,754 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:51:57,332 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:02,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:52:02,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:52:02,354 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:07,465 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:07,850 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:52:12,467 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:17,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:52:17,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:52:17,934 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:52:18,295 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:23,296 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:23,403 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:52:28,006 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:52:28,949 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:32,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:52:32,101 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:52:34,358 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:38,072 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:52:39,706 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:44,708 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:47,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:52:47,101 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:52:48,152 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:52:50,317 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:53,404 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:52:55,405 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:52:58,237 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:53:01,209 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:02,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:53:02,101 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:53:06,261 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:10,314 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:53:11,899 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:16,900 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:17,228 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:53:17,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:53:17,233 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:53:20,389 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:53:22,706 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:23,405 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:53:28,406 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:29,298 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:53:29,300 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:53:29,301 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:53:29,305 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:53:29,443 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:53:30,450 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:53:31,454 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:53:32,228 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:53:32,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:53:32,229 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:53:33,518 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:38,521 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:42,531 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:53:43,845 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:47,231 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:53:47,232 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:53:47,234 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:53:49,448 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:53:51,608 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:53:53,406 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:53:55,407 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:00,409 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:01,728 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:54:02,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:54:02,232 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:54:05,468 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:10,470 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:11,799 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:54:16,154 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:17,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:54:17,231 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:54:21,423 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:21,871 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:54:23,406 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:54:27,408 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:32,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:54:32,231 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:54:32,498 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:33,964 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:54:37,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:43,395 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:44,041 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:54:47,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:54:47,230 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:54:48,444 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:53,408 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:54:53,802 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:54:56,127 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:54:58,803 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:03,291 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:55:03,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:55:03,295 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:55:04,447 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:06,201 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:55:09,541 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:14,543 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:16,272 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:55:18,292 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:55:18,292 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:55:18,293 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:55:20,491 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:23,409 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:55:26,075 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:28,372 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:55:31,075 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:33,292 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:55:33,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:55:33,293 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:55:36,477 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:38,443 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:55:41,790 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:46,792 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:48,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:55:48,293 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:55:48,509 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:55:52,453 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:55:53,409 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:55:55,546 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:55:56,550 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:55:58,287 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:01,579 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:02,585 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:03,292 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:03,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:56:03,293 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:56:08,453 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:12,660 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:13,774 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:18,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:56:18,294 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:56:18,295 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:56:19,463 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:22,739 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:23,409 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:56:25,411 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:30,412 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:31,893 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:33,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:56:33,294 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:56:35,553 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:40,556 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:44,049 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:45,622 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:48,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:56:48,294 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:56:51,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:56:52,955 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 20:56:52,956 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 20:56:52,957 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 20:56:52,960 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 20:56:53,107 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:53,108 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 20:56:53,410 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:56:54,112 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:56:57,411 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:02,412 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:03,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:57:03,294 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:57:04,184 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:57:07,546 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:12,547 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:14,259 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:57:17,788 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:18,293 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:57:18,294 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:57:23,411 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:57:23,412 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:26,366 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:57:29,282 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:33,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:57:33,341 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:57:33,342 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:57:34,545 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:36,443 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:57:39,627 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:44,631 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:46,519 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:57:48,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:57:48,343 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:57:48,385 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:57:50,623 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:53,413 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:57:56,420 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:57:58,593 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:58:01,561 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:03,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:58:03,340 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:58:06,581 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:08,685 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:58:11,929 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:16,931 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:18,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:58:18,341 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:58:18,778 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:58:22,549 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:23,413 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:58:28,310 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:30,859 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:58:33,311 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:33,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:58:33,341 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:58:38,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:40,932 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:58:43,988 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:48,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:58:48,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:58:48,433 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:58:49,603 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:58:50,004 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:58:53,413 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:58:55,414 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:01,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:02,094 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:59:03,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:59:03,432 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:59:06,685 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:12,172 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:59:12,403 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:17,404 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:18,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:59:18,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:59:18,447 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:59:22,235 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:59:22,705 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:23,414 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:59:28,415 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:33,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:33,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:59:33,433 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:59:34,323 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:59:38,603 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:43,951 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:44,398 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:59:48,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 20:59:48,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 20:59:48,433 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 20:59:49,698 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:53,414 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 20:59:54,469 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 20:59:54,823 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 20:59:59,824 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:03,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:00:03,434 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:00:03,434 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:00:04,533 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:00:04,992 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:09,992 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:15,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:16,625 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:00:18,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:00:18,435 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:00:20,637 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:23,417 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:00:24,717 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:00:24,718 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:00:24,719 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:00:24,723 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:00:25,703 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:00:25,703 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:00:25,726 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:26,710 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:00:30,727 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:34,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:00:34,155 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:00:34,156 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:00:36,282 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:36,794 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:00:41,287 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:47,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:48,879 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:00:49,168 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:00:49,169 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:00:49,169 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:00:52,389 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:53,417 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:00:58,128 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:00:58,950 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:01:03,128 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:04,168 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:01:04,169 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:01:04,169 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:01:08,150 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:09,018 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:01:13,151 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:18,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:19,169 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:01:19,170 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:01:19,171 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:01:21,098 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:01:23,414 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:23,417 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:01:28,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:33,176 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:01:34,168 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:01:34,169 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:01:34,441 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:39,442 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:44,443 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:46,273 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:01:49,168 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:01:49,169 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:01:50,427 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:53,419 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:01:55,718 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:01:56,349 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:02:00,718 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:05,065 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:02:05,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:02:05,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:02:06,341 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:08,426 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:02:11,444 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:16,446 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:18,500 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:02:20,065 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:02:20,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:02:20,066 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:02:22,234 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:23,419 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:02:27,421 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:28,580 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:02:32,780 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:35,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:02:35,068 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:02:35,069 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:02:38,158 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:40,667 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:02:43,159 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:48,161 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:50,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:02:50,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:02:50,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:02:50,738 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:02:53,329 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:02:53,420 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:02:58,422 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:00,806 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:03:03,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:05,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:03:05,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:03:05,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:03:09,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:10,879 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:03:14,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:19,100 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:20,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:03:20,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:03:20,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:03:20,963 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:03:23,421 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:03:24,421 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:29,423 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:33,092 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:03:35,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:03:35,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:03:35,067 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:03:35,313 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:40,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:43,162 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:03:45,903 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:50,068 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:03:50,070 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:03:50,071 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:03:51,349 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:03:53,248 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:03:53,421 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:03:56,422 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:01,423 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:01,918 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:04:01,920 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:04:01,921 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:04:01,925 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:04:02,313 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:04:03,319 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:04:04,323 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:04:05,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:04:05,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:04:05,068 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:04:07,313 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:13,198 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:15,408 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:04:18,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:21,265 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:04:21,267 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:04:21,268 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:04:23,421 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:04:23,422 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:24,605 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:04:28,423 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:34,327 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:34,680 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:04:36,265 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:04:36,267 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:04:39,521 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:44,723 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:44,757 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:04:49,731 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:52,758 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:04:52,760 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:04:52,761 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:04:53,423 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:04:55,322 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:04:56,850 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:05:00,323 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:06,116 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:06,925 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:05:07,760 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:05:07,760 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:05:07,760 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:05:11,945 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:16,997 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:05:17,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:22,366 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:23,426 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:05:24,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:05:24,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:05:24,290 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:05:27,062 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:05:27,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:32,671 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:38,188 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:39,153 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:05:39,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:05:39,289 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:05:43,522 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:48,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:49,260 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:05:53,425 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:05:54,426 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:05:55,881 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:05:55,884 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:05:55,885 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:05:59,328 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:06:00,416 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:01,340 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:06:05,428 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:10,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:10,882 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:06:10,883 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:06:10,884 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:06:13,420 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:06:16,386 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:21,388 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:23,426 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:06:23,496 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:06:25,882 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:06:25,882 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:06:25,882 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:06:27,121 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:32,123 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:35,576 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:06:37,428 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:41,245 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:06:41,246 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:06:41,247 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:06:42,475 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:44,660 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:06:48,158 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:53,160 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:06:53,426 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:06:54,761 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:06:56,246 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:06:56,246 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:06:58,456 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:03,458 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:06,847 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:07:08,652 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:11,246 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:07:11,246 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:07:11,247 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:07:14,478 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:16,919 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:07:20,172 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:23,428 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:07:25,222 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:26,247 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:07:26,248 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:07:26,987 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:07:30,496 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:35,498 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:36,245 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:07:36,246 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:07:36,246 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:07:36,249 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:07:37,067 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:07:37,067 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:07:38,072 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:07:41,246 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:41,247 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:07:41,247 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:07:46,511 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:49,149 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:07:51,982 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:53,428 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:07:56,247 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:07:56,248 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:07:56,249 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:07:57,154 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:07:59,221 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:08:02,155 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:07,158 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:09,298 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:08:11,247 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:08:11,248 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:08:12,490 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:17,491 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:19,366 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:08:23,070 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:23,429 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:08:26,812 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:08:26,814 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:08:26,815 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:08:28,185 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:29,440 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:08:33,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:38,291 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:41,523 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:08:41,812 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:08:41,813 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:08:41,813 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:08:43,987 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:49,981 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:51,597 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:08:53,430 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:08:55,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:08:57,899 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:08:57,900 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:08:57,901 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:09:01,194 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:01,679 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:09:06,196 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:11,763 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:09:12,093 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:12,898 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:09:12,898 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:09:17,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:22,774 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:23,431 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:09:23,854 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:09:27,899 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:09:27,900 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:09:27,901 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:09:28,128 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:33,051 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:09:33,318 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:38,318 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:42,898 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:09:42,899 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:09:44,143 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:45,150 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:09:49,144 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:53,430 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:09:54,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:09:55,235 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:09:57,898 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:09:57,899 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:10:00,074 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:05,297 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:10:05,953 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:10,955 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:13,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:10:13,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:10:13,003 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:10:15,366 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:10:16,288 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:21,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:23,431 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:10:27,040 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:27,534 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:10:28,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:10:28,001 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:10:28,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:10:32,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:37,575 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:37,609 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:10:42,576 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:43,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:10:43,004 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:10:43,006 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:10:47,680 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:10:48,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:53,220 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:53,433 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:10:58,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:10:58,002 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:10:58,226 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:10:59,773 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:11:03,226 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:07,105 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:11:07,147 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:11:07,148 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:11:07,152 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:11:07,844 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:11:08,850 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:11:09,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:09,854 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:11:13,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:11:13,004 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:11:14,182 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:19,406 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:19,929 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:11:23,446 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:11:24,447 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:28,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:11:28,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:11:28,004 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:11:30,009 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:11:30,304 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:35,305 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:39,074 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:11:40,484 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:43,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:11:43,003 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:11:46,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:51,171 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:11:51,639 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:53,434 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:11:57,436 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:11:58,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:11:58,004 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:12:01,243 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:12:02,820 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:07,822 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:11,322 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:12:13,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:12:13,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:13,004 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:12:18,202 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:21,395 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:12:23,435 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:12:23,436 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:28,437 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:29,089 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:12:29,096 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:12:29,098 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:12:33,484 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:12:33,793 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:38,795 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:43,556 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:12:44,089 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:12:44,091 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:12:44,092 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:12:44,273 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:49,275 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:53,435 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:12:53,651 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:12:54,436 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:12:59,091 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:12:59,114 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:12:59,134 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:13:00,410 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:03,734 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:13:06,249 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:11,251 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:14,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:13:14,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:13:14,091 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:13:15,814 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:13:16,332 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:21,334 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:23,436 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:13:25,892 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:13:26,844 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:29,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:13:29,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:13:29,091 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:13:32,294 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:35,979 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:13:38,025 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:43,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:44,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:13:44,091 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:13:44,092 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:13:46,050 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:13:48,302 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:53,304 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:53,438 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:13:56,119 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:13:58,415 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:13:59,090 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:13:59,091 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:13:59,091 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:14:04,302 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:08,241 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:14:10,241 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:14,542 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:14:14,544 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:14:14,545 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:14:15,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:17,313 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:14:21,175 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:23,440 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:14:26,443 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:27,388 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:14:29,541 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:14:29,542 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:14:31,750 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:36,752 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:37,086 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:14:37,088 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:14:37,088 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:14:37,091 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:14:37,457 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:14:38,462 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:14:39,468 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:14:42,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:46,165 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:14:46,166 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:14:46,168 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:14:47,441 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:49,551 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:14:52,777 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:53,439 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:14:58,441 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:14:59,656 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:15:01,165 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:15:01,165 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:15:01,166 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:15:04,367 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:10,105 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:11,749 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:15:15,105 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:16,166 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:15:16,167 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:15:16,168 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:15:20,427 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:21,816 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:15:23,440 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:15:25,441 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:31,050 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:31,165 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:15:31,166 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:15:31,894 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:15:36,326 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:41,387 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:41,957 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:15:46,166 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:15:46,166 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:15:46,167 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:15:47,347 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:52,032 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:15:52,848 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:15:53,440 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:15:58,442 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:01,166 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:16:01,167 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:16:01,168 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:16:03,705 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:04,101 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:16:08,706 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:14,169 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:16:14,434 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:16,166 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:16:16,167 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:16:20,406 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:23,453 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:16:24,244 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:16:25,454 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:30,456 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:31,493 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:16:31,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:16:31,495 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:16:34,316 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:16:35,850 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:40,851 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:46,211 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:46,493 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:16:46,494 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:16:46,501 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:16:51,754 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:16:53,442 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:16:55,576 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:16:57,568 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:02,569 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:02,874 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:17:02,875 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:17:05,662 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:17:08,395 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:13,396 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:17,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:17:17,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:17:17,878 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:17:19,043 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:21,781 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:17:23,442 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:17:24,443 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:29,445 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:31,861 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:17:32,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:17:32,875 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:17:35,052 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:40,054 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:41,935 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:17:45,209 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:47,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:17:47,875 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:17:51,116 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:17:53,443 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:17:54,016 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:17:56,444 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:01,446 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:02,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:18:02,876 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:18:04,089 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:18:07,155 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:12,156 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:12,270 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:18:12,271 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:18:12,272 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:18:12,274 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:18:12,329 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:18:13,335 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:18:14,340 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:18:17,276 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:17,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:18:17,876 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:18:23,007 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:23,444 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:18:24,407 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:18:28,445 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:32,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:18:32,877 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:18:32,878 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:18:33,807 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:36,494 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:18:38,808 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:43,810 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:46,562 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:18:47,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:18:47,876 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:18:49,107 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:53,446 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:18:54,448 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:18:56,625 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:18:59,853 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:03,898 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:19:03,899 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:19:03,901 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:19:05,107 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:06,686 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:19:10,147 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:15,149 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:17,761 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:19:18,898 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:19:18,899 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:19:18,899 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:19:21,147 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:23,448 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:19:26,439 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:27,837 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:19:31,440 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:34,705 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:19:34,706 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:19:34,708 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:19:37,061 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:37,939 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:19:42,062 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:47,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:48,011 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:19:49,705 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:19:49,705 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:19:52,892 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:53,447 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:19:57,947 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:19:58,080 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:20:02,949 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:05,760 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:20:05,761 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:20:08,143 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:08,212 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:20:13,156 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:18,602 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:20,300 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:20:20,760 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:20:20,760 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:20:23,448 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:20:24,449 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:29,770 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:30,370 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:20:34,771 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:36,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:20:36,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:20:36,671 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:20:40,106 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:40,548 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:20:45,106 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:50,631 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:20:50,857 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:20:51,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:20:51,669 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:20:53,448 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:20:56,449 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:01,450 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:06,451 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:06,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:21:06,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:21:06,671 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:21:11,907 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:12,796 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:21:16,908 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:21,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:21:21,715 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:21:21,716 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:21:21,949 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:23,449 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:21:24,878 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:21:27,450 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:32,786 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:33,947 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:21:36,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:21:36,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:21:36,671 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:21:37,909 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:43,507 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:44,022 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:21:48,508 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:51,860 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:21:51,860 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:21:51,862 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:21:53,423 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:21:53,424 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:21:53,425 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:21:53,443 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:21:53,449 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:21:54,140 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:21:54,140 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:21:54,450 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:21:55,145 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:21:59,451 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:04,681 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:06,222 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:22:06,860 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:22:06,860 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:22:10,039 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:15,942 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:16,295 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:22:20,944 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:23,058 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:22:23,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:22:23,072 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:22:23,451 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:22:26,380 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:22:26,835 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:31,837 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:36,452 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:22:36,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:38,057 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:22:38,058 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:22:38,059 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:22:42,317 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:46,532 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:22:48,188 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:53,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:22:53,453 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:22:54,306 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:22:54,307 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:22:54,311 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:22:58,674 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:22:58,943 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:03,944 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:08,761 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:23:09,306 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:23:09,307 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:23:09,489 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:14,491 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:18,845 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:23:20,463 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:23,453 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:23:24,306 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:23:24,306 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:23:25,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:30,937 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:23:31,298 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:36,299 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:39,306 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:23:39,307 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:23:41,008 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:23:41,458 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:46,459 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:51,791 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:23:53,081 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:23:53,454 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:23:54,306 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:23:54,307 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:23:57,497 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:02,150 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:24:02,854 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:07,855 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:10,377 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:24:10,378 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:24:10,380 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:24:12,214 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:24:13,672 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:18,673 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:23,454 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:24:24,291 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:24:24,455 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:25,377 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:24:25,378 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:24:25,378 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:24:29,591 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:34,375 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:24:34,606 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:39,609 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:41,480 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:24:41,481 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:24:41,481 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:24:44,448 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:24:44,869 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:49,870 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:53,455 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:24:54,522 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:24:55,455 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:24:56,480 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:24:56,481 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:24:56,482 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:25:00,696 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:05,811 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:06,608 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:25:10,813 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:12,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:25:12,672 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:25:12,674 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:25:16,019 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:16,692 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:25:21,021 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:23,455 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:25:24,157 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:25:24,158 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:25:24,159 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:25:24,162 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:25:24,761 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:25:25,766 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:25:26,165 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:26,772 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:25:27,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:25:27,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:25:27,669 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:25:31,871 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:36,847 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:25:37,248 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:42,249 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:43,438 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:25:43,440 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:25:43,443 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:25:46,917 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:25:47,794 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:52,795 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:25:53,458 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:25:56,997 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:25:58,438 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:25:58,439 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:25:58,439 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:25:58,619 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:03,621 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:07,063 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:26:08,689 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:13,691 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:14,652 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:26:14,652 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:26:14,656 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:26:19,045 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:19,163 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:26:23,472 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:26:24,474 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:29,237 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:26:29,652 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:26:29,652 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:26:29,653 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:26:29,807 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:34,809 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:39,311 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:26:40,471 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:45,472 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:45,812 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:26:45,814 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:26:45,815 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:26:48,386 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:26:51,280 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:53,459 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:26:56,460 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:26:58,456 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:27:00,813 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:27:00,814 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:27:00,814 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:27:02,052 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:07,054 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:08,519 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:27:12,916 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:15,814 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:27:15,816 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:27:18,087 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:20,602 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:27:23,303 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:23,459 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:27:28,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:30,667 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:27:30,816 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:27:30,817 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:27:30,817 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:27:33,979 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:38,981 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:40,745 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:27:44,891 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:45,815 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:27:45,816 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:27:50,039 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:27:52,822 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:27:53,460 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:27:55,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:00,462 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:00,817 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:28:00,817 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:28:02,889 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:28:05,643 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:10,644 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:14,972 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:28:15,815 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:28:15,816 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:15,817 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:28:21,044 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:23,461 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:28:25,042 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:28:26,463 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:31,465 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:31,558 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:28:31,559 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:28:31,561 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:28:35,107 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:28:37,030 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:42,032 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:45,189 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:28:46,570 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:28:46,571 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:28:46,572 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:28:47,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:52,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:28:53,462 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:28:54,235 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:28:54,236 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:28:54,237 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:28:54,239 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:28:54,244 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:28:55,259 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:28:57,268 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:28:58,241 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:03,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:29:03,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:29:03,103 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:29:03,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:07,354 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:29:08,642 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:13,655 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:17,423 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:29:18,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:29:18,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:29:19,279 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:23,465 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:29:24,467 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:27,498 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:29:30,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:34,429 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:29:34,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:29:34,431 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:29:35,644 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:36,660 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:29:40,823 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:45,825 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:48,751 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:29:49,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:29:49,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:29:49,432 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:29:51,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:53,466 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:29:56,994 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:29:58,837 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:30:01,995 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:04,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:30:04,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:30:04,434 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:30:07,688 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:08,903 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:30:12,688 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:17,690 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:18,983 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:30:19,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:30:19,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:30:19,431 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:30:22,699 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:23,466 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:30:28,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:29,069 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:30:33,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:34,430 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:30:34,434 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:30:38,654 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:41,241 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:30:43,655 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:48,656 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:49,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:30:49,432 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:30:51,334 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:30:53,467 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:30:54,468 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:30:59,469 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:03,414 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:31:04,431 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:31:04,431 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:31:04,594 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:09,596 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:13,485 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:31:15,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:20,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:20,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:31:20,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:31:20,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:31:23,479 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:31:25,567 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:31:26,480 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:31,482 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:34,733 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:31:35,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:31:35,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:31:35,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:31:37,173 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:42,175 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:44,810 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:31:47,717 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:52,211 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:31:52,212 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:31:52,214 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:31:53,468 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:31:53,469 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:31:54,879 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:31:58,594 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:03,596 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:04,962 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:32:07,211 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:32:07,212 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:32:09,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:14,463 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:17,049 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:32:19,571 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:23,458 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:32:23,459 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:32:23,460 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:32:23,646 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:32:24,649 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:24,848 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:32:24,849 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:32:24,850 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:32:24,853 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:32:25,110 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:32:26,126 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:32:27,133 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:32:29,855 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:34,856 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:37,223 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:32:38,459 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:32:38,460 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:32:40,683 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:45,684 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:51,328 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:32:51,401 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:32:53,458 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:32:53,459 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:32:53,628 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:32:56,630 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:01,793 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:03,412 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:33:06,794 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:09,537 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:33:09,538 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:33:09,540 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:33:11,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:13,481 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:33:16,889 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:22,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:23,470 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:33:23,617 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:33:24,538 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:33:24,538 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:33:27,762 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:33,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:35,709 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:33:38,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:39,538 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:33:39,538 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:33:43,792 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:45,786 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:33:49,063 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:53,473 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:33:54,475 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:33:54,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:33:54,540 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:33:54,541 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:33:57,894 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:33:59,729 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:04,731 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:09,000 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:34:09,538 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:34:09,539 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:34:10,703 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:15,705 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:21,081 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:34:21,304 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:23,473 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:34:24,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:34:24,540 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:34:26,733 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:31,152 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:34:32,175 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:37,176 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:39,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:34:39,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:34:39,667 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:34:42,324 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:43,238 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:34:47,325 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:52,326 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:34:53,473 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:34:54,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:34:54,665 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:34:55,352 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:34:57,827 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:02,828 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:05,431 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:35:08,375 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:09,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:35:09,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:35:09,666 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:35:13,834 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:17,528 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:35:19,345 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:23,474 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:35:24,477 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:24,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:35:24,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:35:24,672 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:35:27,610 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:35:30,171 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:35,172 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:37,694 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:35:39,665 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:35:39,666 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:35:40,881 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:45,883 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:49,846 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:35:51,389 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:53,474 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:35:54,667 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:35:54,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:35:54,671 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:35:56,923 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:35:59,938 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:36:02,080 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:07,081 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:09,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:36:09,201 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:36:09,201 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:36:09,204 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:36:09,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:36:09,666 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:36:10,012 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:36:10,013 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:36:12,021 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:36:12,822 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:17,823 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:22,084 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:36:23,475 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:36:23,476 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:24,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:36:24,666 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:36:28,863 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:31,164 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:36:34,178 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:39,180 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:39,667 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:36:39,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:36:39,668 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:36:41,248 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:36:44,628 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:49,630 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:53,322 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:36:53,477 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:36:54,666 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:36:54,667 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:36:54,867 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:36:59,869 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:03,399 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:37:05,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:10,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:11,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:37:11,259 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:37:11,261 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:37:13,476 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:37:15,545 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:20,546 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:23,477 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:37:23,542 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:37:26,257 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:37:26,257 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:26,258 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:37:31,506 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:35,618 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:37:37,002 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:41,853 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:37:41,855 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:37:41,857 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:37:42,049 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:45,684 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:37:47,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:52,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:37:53,477 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:37:55,762 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:37:56,853 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:37:56,854 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:37:58,064 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:03,066 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:05,863 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:38:08,735 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:11,853 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:38:11,854 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:38:14,107 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:18,061 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:38:19,409 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:23,479 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:38:24,481 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:26,855 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:38:26,856 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:38:30,046 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:30,141 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:38:35,048 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:40,221 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:38:40,636 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:41,854 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:38:41,855 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:38:46,051 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:50,300 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:38:51,350 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:53,479 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:38:56,480 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:38:57,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:38:57,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:38:57,741 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:39:01,376 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:39:02,146 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:07,148 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:11,459 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:39:12,450 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:12,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:39:12,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:39:12,740 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:39:17,921 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:21,536 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:39:23,235 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:23,479 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:39:27,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:39:27,743 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:39:27,749 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:39:28,997 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:31,617 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:39:34,926 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:39,928 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:41,850 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:39:41,852 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:39:41,853 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:39:41,857 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:39:42,692 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:39:42,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:39:42,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:39:42,742 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:39:43,697 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:39:44,703 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:39:45,918 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:50,920 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:53,480 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:39:53,784 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:39:56,482 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:39:57,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:39:57,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:39:57,741 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:40:01,982 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:05,903 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:40:07,672 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:12,685 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:13,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:40:13,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:40:13,256 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:40:15,973 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:40:18,479 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:23,482 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:23,483 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:40:26,045 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:40:28,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:40:28,253 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:40:29,476 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:34,479 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:38,161 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:40:39,557 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:43,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:40:43,253 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:40:45,509 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:50,278 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:40:50,589 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:53,482 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:40:56,484 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:40:58,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:40:58,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:40:58,255 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:41:02,346 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:02,359 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:41:07,348 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:12,453 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:41:12,704 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:13,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:41:13,254 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:41:18,470 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:23,494 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:41:23,495 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:24,545 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:41:28,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:41:28,254 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:41:29,428 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:33,646 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:41:34,713 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:39,715 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:43,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:41:43,255 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:41:43,256 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:41:45,188 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:45,732 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:41:50,189 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:53,485 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:41:55,850 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:41:56,089 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:41:58,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:41:58,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:41:58,255 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:42:01,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:06,443 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:07,971 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:42:11,799 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:13,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:42:13,255 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:42:17,427 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:20,129 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:42:23,003 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:23,485 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:42:28,483 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:42:28,486 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:42:28,488 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:28,489 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:42:32,207 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:42:33,991 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:38,993 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:42,293 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:42:43,483 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:42:43,484 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:42:44,676 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:49,678 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:52,355 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:42:53,486 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:42:55,487 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:42:59,884 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:42:59,885 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:42:59,888 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:43:01,140 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:02,445 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:43:06,141 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:11,142 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:14,526 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:43:14,884 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:43:14,885 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:43:17,103 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:22,104 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:23,245 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:43:23,247 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:43:23,247 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:43:23,249 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:43:23,493 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:43:23,612 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:43:24,619 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:43:27,508 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:29,885 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:43:29,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:43:29,887 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:43:33,134 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:36,702 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:43:38,400 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:43,403 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:44,885 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:43:44,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:43:44,888 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:43:47,785 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:43:49,179 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:53,493 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:43:54,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:57,930 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:43:59,885 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:43:59,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:43:59,886 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:43:59,887 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:44:05,116 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:10,117 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:14,885 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:44:14,886 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:44:16,110 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:21,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:23,493 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:44:26,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:30,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:44:30,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:44:30,435 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:44:31,699 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:34,168 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:44:37,210 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:42,211 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:45,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:44:45,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:44:45,433 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:44:46,244 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:44:47,634 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:52,636 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:44:53,495 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:44:56,319 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:44:58,181 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:00,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:45:00,432 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:45:00,433 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:45:03,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:06,386 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:45:09,115 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:14,117 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:15,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:45:15,435 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:45:15,436 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:45:18,473 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:45:19,676 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:23,497 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:45:25,500 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:28,543 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:45:30,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:45:30,433 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:45:30,434 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:45:30,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:35,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:38,616 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:45:41,524 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:46,525 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:46,737 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:45:46,738 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:45:46,740 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:45:50,697 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:45:52,156 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:45:53,497 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:45:57,498 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:00,768 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:46:01,738 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:46:01,785 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:46:01,786 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:46:02,999 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:08,001 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:10,848 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:46:13,161 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:16,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:46:16,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:46:16,742 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:46:18,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:22,958 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:46:23,497 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:46:24,498 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:29,499 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:31,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:46:31,741 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:46:32,065 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:46:34,941 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:39,942 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:44,943 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:46,650 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:46:46,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:46:46,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:46:46,741 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:46:50,929 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:53,511 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:46:56,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:46:56,724 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:47:01,068 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:01,739 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:47:01,740 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:47:06,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:08,810 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:47:11,358 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:16,360 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:16,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:47:16,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:47:16,742 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:47:18,903 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:47:22,222 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:23,499 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:47:27,522 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:29,032 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:47:29,035 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:47:29,036 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:47:29,040 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:47:30,017 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:47:31,022 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:47:31,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:47:31,740 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:47:32,900 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:37,902 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:42,115 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:47:43,193 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:46,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:47:46,740 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:47:48,952 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:52,194 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:47:53,500 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:47:54,501 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:47:59,503 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:01,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:48:01,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:48:01,742 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:48:04,303 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:48:05,040 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:10,032 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:14,381 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:48:15,704 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:16,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:48:16,741 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:48:20,951 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:23,502 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:48:26,563 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:48:26,860 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:31,740 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:48:31,741 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:48:31,950 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:37,504 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:38,657 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:48:42,505 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:46,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:48:46,742 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:48:46,744 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:48:47,621 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:48,738 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:48:52,622 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:48:53,502 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:48:58,403 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:00,821 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:49:01,742 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:49:01,742 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:49:03,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:08,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:10,890 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:49:14,144 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:16,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:49:16,742 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:49:19,955 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:22,979 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:49:23,503 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:49:25,504 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:30,505 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:31,741 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:49:31,742 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:49:35,980 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:39,093 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:49:41,136 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:46,138 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:46,742 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:49:46,743 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:49:49,168 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:49:52,131 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:53,504 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:49:57,505 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:49:59,244 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:50:01,742 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:50:01,743 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:50:02,907 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:07,908 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:10,328 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:50:13,837 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:18,087 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:50:18,091 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:50:19,309 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:20,397 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:50:23,505 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:50:24,506 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:29,507 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:32,477 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:50:33,129 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:50:33,130 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:50:33,131 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:50:35,318 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:40,320 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:42,546 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:50:46,188 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:48,130 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:50:48,131 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:50:51,312 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:50:53,506 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:50:54,625 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:50:56,509 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:01,510 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:03,129 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:51:03,130 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:51:04,701 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:51:07,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:12,316 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:13,039 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:51:13,041 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:51:13,042 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:51:13,046 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:51:13,769 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:51:14,776 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:51:18,049 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:18,130 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:51:18,130 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:51:18,131 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:51:23,302 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:23,512 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:51:24,840 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:51:29,111 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:33,667 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:51:33,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:51:33,669 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:51:34,841 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:36,928 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:51:39,843 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:44,845 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:47,124 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:51:48,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:51:48,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:51:48,669 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:51:49,851 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:51:53,528 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:51:55,529 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:00,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:03,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:52:03,715 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:52:03,716 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:52:05,931 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:10,932 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:14,670 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:52:16,337 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:18,668 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:52:18,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:52:18,671 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:52:21,928 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:23,507 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:52:24,764 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:52:27,508 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:32,510 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:33,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:52:33,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:52:33,670 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:52:34,834 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:52:37,828 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:42,830 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:44,924 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:52:48,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:48,669 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:52:48,670 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:52:48,671 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:52:53,510 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:52:53,512 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:52:57,001 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:52:58,736 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:03,738 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:03,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:53:03,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:53:03,971 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:53:07,087 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:53:09,220 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:14,222 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:17,153 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:53:18,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:53:18,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:53:18,970 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:53:20,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:23,511 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:53:25,514 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:27,224 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:53:30,860 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:34,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:53:34,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:53:34,970 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:53:36,176 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:37,302 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:53:41,366 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:46,367 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:49,379 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:53:49,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:53:49,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:53:49,969 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:53:52,187 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:53,511 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:53:57,466 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:53:59,456 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:54:02,466 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:04,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:54:04,971 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:54:04,973 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:54:07,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:09,521 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:54:12,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:17,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:19,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:54:19,970 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:54:21,599 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:54:23,174 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:23,512 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:54:28,513 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:30,708 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:54:33,573 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:34,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:54:34,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:54:34,970 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:54:38,764 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:40,800 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:54:43,765 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:48,767 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:49,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:54:49,971 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:54:50,866 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:54:53,513 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:54:54,514 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:59,516 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:54:59,682 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:54:59,684 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:54:59,685 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:54:59,690 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:54:59,923 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:55:00,928 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:55:01,933 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:55:04,693 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:04,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:55:04,970 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:55:09,830 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:11,022 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:55:14,831 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:19,832 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:19,971 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:55:19,972 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:55:19,972 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:55:21,095 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:55:23,513 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:55:25,513 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:30,515 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:33,172 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:55:34,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:55:34,971 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:55:36,149 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:41,151 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:43,248 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:55:46,190 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:50,087 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:55:50,087 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:55:50,088 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:55:51,311 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:55:53,327 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:55:53,513 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:55:56,515 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:01,516 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:03,412 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:56:05,087 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:56:05,088 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:56:07,270 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:12,272 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:15,519 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:56:17,704 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:20,088 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:56:20,089 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:56:20,091 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:56:23,274 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:23,514 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:56:25,588 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:56:28,515 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:33,518 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:35,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:56:35,145 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:56:35,146 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:56:37,705 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:56:39,387 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:44,389 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:47,779 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:56:49,415 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:50,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:56:50,099 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:56:53,517 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:56:54,519 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:56:56,857 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:56:59,749 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:04,758 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:05,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:57:05,101 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:57:06,972 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:57:10,525 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:15,527 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:19,075 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:57:20,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:57:20,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:57:21,289 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:23,516 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:57:26,518 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:29,144 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:57:31,789 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:36,745 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:57:36,748 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:57:36,749 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:57:36,916 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:39,220 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:57:42,901 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:47,903 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:49,298 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:57:51,745 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:57:51,745 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:57:52,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:57:53,517 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:57:58,519 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:01,411 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:58:04,495 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:06,746 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:58:06,747 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:58:06,749 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:58:09,947 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:11,500 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:58:15,220 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:20,221 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:21,746 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:58:21,748 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:58:23,519 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:58:23,591 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:58:25,520 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:30,522 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:32,255 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 21:58:32,257 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 21:58:32,257 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 21:58:32,260 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 21:58:32,657 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 21:58:33,666 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:58:36,262 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:36,745 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:58:36,768 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:58:41,928 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:46,930 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:49,987 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:58:51,747 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:58:51,748 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:58:51,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:53,519 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:58:57,521 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:58:59,137 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:59:02,753 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:07,754 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:07,957 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:59:07,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:59:07,960 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:59:11,244 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:59:13,312 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:18,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:21,311 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:59:22,957 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:59:22,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:59:22,958 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:59:23,520 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:59:23,521 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:28,522 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:33,432 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:59:34,161 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:37,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:59:37,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:59:37,958 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:59:39,195 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:43,497 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:59:44,916 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:49,918 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 21:59:52,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 21:59:52,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 21:59:52,959 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 21:59:53,521 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 21:59:53,574 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 21:59:55,523 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:00,525 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:03,658 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:00:05,926 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:07,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:00:07,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:00:07,959 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:00:11,120 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:13,731 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:00:16,947 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:21,949 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:22,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:00:22,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:00:22,961 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:00:23,522 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:00:25,814 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:00:27,523 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:32,525 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:35,877 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:00:37,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:00:37,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:00:37,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:37,960 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:00:43,227 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:47,969 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:00:48,262 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:52,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:00:52,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:00:52,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:00:53,523 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:00:53,526 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:00:58,043 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:00:58,934 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:03,936 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:07,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:01:07,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:01:07,960 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:01:08,116 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:01:09,216 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:14,218 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:18,198 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:01:19,362 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:22,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:01:22,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:01:22,960 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:01:23,523 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:01:24,525 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:29,286 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:01:30,435 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:35,436 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:37,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:01:37,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:01:37,961 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:01:39,364 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:01:41,253 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:46,254 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:51,472 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:01:52,201 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:01:52,959 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:01:52,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:01:52,961 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:01:53,524 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:01:57,526 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:01,590 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:02:02,553 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:07,575 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:07,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:02:07,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:02:07,965 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:02:11,515 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 22:02:11,517 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 22:02:11,518 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 22:02:11,520 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 22:02:11,661 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 22:02:12,715 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:13,675 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:02:15,685 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:02:18,695 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:22,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:02:22,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:02:22,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:02:23,525 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:02:24,527 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:29,527 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:34,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:37,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:02:37,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:02:37,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:02:40,225 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:45,226 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:50,227 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:02:52,960 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:02:52,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:02:52,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:02:53,525 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:02:55,526 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:00,527 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:05,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:07,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:03:07,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:03:07,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:03:11,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:16,200 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:21,201 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:22,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:03:22,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:03:22,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:03:23,526 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:03:26,527 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:31,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:36,529 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:37,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:03:37,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:03:37,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:03:42,141 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:47,142 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:52,143 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:03:52,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:03:52,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:03:52,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:03:53,528 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:03:57,529 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:02,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:07,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:07,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:04:07,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:04:07,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:04:13,146 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:18,147 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:22,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:04:22,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:04:22,963 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:04:23,147 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:23,529 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:04:28,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:33,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:37,961 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:04:37,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:04:37,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:04:39,212 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:44,213 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:49,214 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:52,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:04:52,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:04:52,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:04:53,530 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:04:54,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:04:59,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:04,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:07,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:05:07,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:05:07,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:05:10,218 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:15,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:20,220 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:22,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:05:22,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:05:22,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:05:23,529 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:05:25,530 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:30,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:35,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:37,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:05:37,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:05:37,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:05:41,213 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:46,214 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:51,215 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:05:52,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:05:52,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:05:52,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:05:53,530 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:05:56,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:01,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:06,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:07,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:06:07,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:06:07,964 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:06:12,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:17,220 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:22,221 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:22,962 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:06:22,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:06:22,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:06:23,530 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:06:27,531 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:32,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:37,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:37,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:06:37,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:06:37,965 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:06:43,127 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:48,128 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:52,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:06:52,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:06:52,965 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:06:53,531 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:06:53,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:06:58,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:03,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:07,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:07:07,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:07:07,966 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:07:09,183 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:14,184 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:19,185 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:22,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:07:22,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:07:22,965 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:07:23,532 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:07:24,532 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:29,533 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:34,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:37,963 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:07:37,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:07:37,966 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:07:40,204 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:45,207 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:50,208 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:07:52,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:07:52,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:07:52,966 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:07:53,532 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:07:55,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:00,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:05,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:07,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:08:07,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:08:07,967 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:08:11,161 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:16,162 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:21,162 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:22,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:08:22,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:08:22,966 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:08:23,533 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:08:26,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:31,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:36,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:37,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:08:37,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:08:37,967 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:08:42,218 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:47,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:52,219 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:08:52,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:08:52,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:08:52,966 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:08:53,533 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:08:57,534 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:02,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:07,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:07,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:09:07,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:09:07,967 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:09:13,135 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:18,136 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:22,964 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:09:22,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:09:22,967 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:09:23,226 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:23,534 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:09:28,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:33,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:37,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:09:37,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:09:37,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:09:39,196 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:44,197 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:49,198 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:52,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:09:52,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:09:52,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:09:53,534 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:09:54,535 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:09:59,536 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:04,538 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:07,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:10:07,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:10:07,967 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:10:10,237 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:15,238 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:20,239 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:22,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:10:22,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:10:22,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:10:23,536 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:10:25,537 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:30,538 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:35,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:37,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:10:37,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:10:37,967 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:10:41,145 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:46,146 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:51,146 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:10:52,965 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:10:52,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:10:52,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:10:53,537 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:10:56,537 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:01,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:06,540 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:07,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:11:07,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:11:07,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:11:12,193 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:17,194 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:22,194 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:22,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:11:22,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:11:22,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:11:23,537 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:11:27,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:32,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:37,540 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:37,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:11:37,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:11:37,969 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:11:43,229 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:48,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:52,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:11:52,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:11:52,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:11:53,538 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:11:53,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:11:58,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:03,540 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:07,966 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:12:07,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:12:07,968 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:12:09,198 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:14,199 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:19,200 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:22,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:12:22,970 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:12:22,971 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:12:23,538 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:12:24,547 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:29,547 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:34,548 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:37,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:12:37,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:12:37,969 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:12:40,152 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:45,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:50,154 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:12:52,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:12:52,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:12:52,969 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:12:53,539 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:12:55,539 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:00,541 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:05,542 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:07,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:13:07,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:13:07,969 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:13:11,264 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:16,265 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:21,266 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:22,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:13:22,968 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:13:22,969 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:13:23,540 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:13:26,542 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:31,543 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:34,059 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 22:13:34,062 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 22:13:34,062 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 22:13:34,065 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 22:13:34,446 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 22:13:36,458 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:13:37,067 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:37,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:13:37,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:13:37,969 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:13:42,153 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:47,154 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:52,155 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:13:52,967 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:13:52,969 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:13:52,970 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:13:53,540 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:13:57,541 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:02,542 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:04,663 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:14:08,298 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:14:08,367 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:09,486 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:14:09,487 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:14:13,742 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:18,743 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:23,428 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:14:23,464 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:14:23,465 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:14:23,708 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:14:24,339 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:29,340 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:34,341 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:38,635 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:14:38,685 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:14:38,686 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:14:40,043 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:45,044 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:50,707 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:50,904 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:14:53,541 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:14:54,279 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:14:56,542 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:14:58,200 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:14:58,200 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:15:00,954 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:15:02,457 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:05,638 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:15:07,458 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:12,460 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:13,418 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:15:17,460 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:20,990 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:15:20,990 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:15:23,086 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:15:23,185 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:23,541 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:15:28,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:15:28,420 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:28,420 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:15:33,581 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:38,582 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:43,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:15:43,420 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:15:43,649 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:48,649 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:53,542 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:15:54,543 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:15:58,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:15:58,419 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:15:59,579 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:04,580 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:09,580 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:13,420 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:16:13,420 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:16:14,582 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:19,583 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:23,542 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:16:25,543 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:28,419 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:16:28,420 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:16:30,609 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:32,440 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:16:36,023 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:41,024 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:44,384 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:16:44,385 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:16:44,387 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:16:46,535 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:16:46,907 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:51,908 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:53,544 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:16:57,527 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:16:58,631 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:16:59,383 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:16:59,384 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:17:02,576 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:07,577 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:10,719 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:17:12,639 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:14,383 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:17:14,384 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:17:18,623 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:22,807 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:17:23,544 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:17:24,545 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:29,385 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:17:29,386 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:17:29,387 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:17:29,655 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:34,885 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:17:34,998 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:39,999 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:44,493 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:17:44,495 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:17:44,496 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:17:45,722 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:46,971 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:17:50,905 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:53,545 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:17:56,548 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:17:59,052 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:17:59,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:17:59,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:17:59,495 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:18:01,663 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:06,664 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:11,134 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:18:12,053 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:14,493 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:18:14,494 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:18:14,495 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:18:17,700 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:22,735 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:23,240 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:18:23,546 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:18:28,548 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:30,700 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:18:30,701 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:18:30,703 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:18:33,318 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:18:33,947 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:38,948 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:44,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:45,441 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:18:45,700 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:18:45,701 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:18:49,907 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:53,547 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:18:55,874 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:18:57,550 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:19:00,700 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:19:00,701 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:19:00,701 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:19:00,902 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:05,904 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:08,628 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:19:11,634 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:16,391 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:19:16,393 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:19:16,394 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:19:17,570 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:18,702 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:19:23,051 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:23,548 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:19:28,549 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:30,793 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:19:31,391 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:19:31,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:19:31,393 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:19:33,582 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:38,583 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:42,891 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:19:44,021 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:46,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:19:46,393 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:19:49,594 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:19:53,549 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:19:55,002 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:19:55,553 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:00,553 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:01,393 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:20:01,394 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:20:01,396 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:20:06,255 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:07,091 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:20:11,256 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:16,258 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:16,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:20:16,393 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:20:19,204 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:20:21,271 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:23,550 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:20:26,551 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:31,392 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:20:31,393 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:20:31,609 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:36,466 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 22:20:36,469 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 22:20:36,470 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 22:20:36,473 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 22:20:37,336 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:20:37,337 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 22:20:37,475 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:39,348 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:20:42,476 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:46,460 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:20:46,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:20:46,462 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:20:47,672 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:49,432 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:20:53,121 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:20:53,550 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:20:58,553 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:01,460 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:21:01,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:21:01,461 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:21:01,571 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:21:03,654 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:08,656 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:13,679 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:21:14,230 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:16,460 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:21:16,461 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:21:16,464 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:21:19,723 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:23,551 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:21:25,552 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:25,759 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:21:30,554 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:32,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:21:32,358 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:21:32,358 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:21:34,853 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:21:35,957 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:40,958 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:46,059 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:46,939 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:21:47,356 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:21:47,357 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:21:51,558 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:21:53,553 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:21:57,016 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:21:57,149 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:02,150 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:02,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:22:02,357 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:22:02,358 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:22:07,579 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:09,147 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:22:13,154 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:17,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:22:17,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:22:17,878 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:22:19,096 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:21,233 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:22:23,553 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:22:24,554 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:29,556 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:31,311 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:22:32,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:22:32,876 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:22:35,071 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:40,073 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:41,391 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:22:45,629 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:47,874 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:22:47,877 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:22:51,070 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:22:53,555 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:22:55,714 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:22:56,556 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:01,558 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:02,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:23:02,877 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:23:06,751 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:07,837 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:23:11,753 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:17,729 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:17,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:23:17,876 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:23:17,909 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:23:23,126 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:23,557 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:23:26,975 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:23:28,687 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:32,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:23:32,876 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:23:34,133 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:39,065 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:23:39,866 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:44,867 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:47,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:23:47,878 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:23:47,879 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:23:50,218 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:23:51,161 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:23:53,557 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:23:55,558 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:01,301 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:24:01,528 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:02,875 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:24:02,876 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:24:07,131 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:13,124 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:13,393 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:24:17,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:24:17,877 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:24:19,052 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:23,558 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:24:23,569 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: partial_history +2023-11-22 22:24:23,570 DEBUG SenderThread:2219461 [sender.py:send():380] send: history +2023-11-22 22:24:23,572 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: summary_record +2023-11-22 22:24:23,576 INFO SenderThread:2219461 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-22 22:24:24,483 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/wandb-summary.json +2023-11-22 22:24:24,578 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:25,489 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:24:29,580 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:32,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:24:32,878 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:24:35,211 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:35,558 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:24:40,212 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:46,074 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:47,696 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:24:47,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:24:47,877 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:24:52,058 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:53,560 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:24:57,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:24:59,784 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:25:02,366 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:02,876 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:25:02,877 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:25:08,143 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:09,856 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:25:13,809 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:18,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: keepalive +2023-11-22 22:25:18,099 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:25:18,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:25:19,281 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:21,936 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:25:23,560 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:25:24,563 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:29,565 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:32,010 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:25:33,096 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:25:33,096 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:25:35,353 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:40,355 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:44,110 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:25:45,676 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:48,096 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:25:48,097 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:25:51,358 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:25:53,172 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:25:53,561 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:25:56,562 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:01,564 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:03,097 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:26:03,097 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:26:05,258 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:26:07,543 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:12,545 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:15,347 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:26:17,662 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:18,096 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:26:18,097 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:26:23,267 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:23,576 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:26:27,452 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:26:28,928 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:33,096 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:26:33,097 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:26:34,365 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:39,586 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:26:40,300 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:45,301 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:48,097 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:26:48,098 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:26:49,660 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:26:50,403 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:26:53,563 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:26:55,564 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:00,696 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:01,774 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:27:03,096 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:27:03,097 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:27:06,318 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:11,604 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:11,839 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:27:16,613 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:18,098 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:27:18,100 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:27:22,252 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:23,564 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:27:23,933 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:27:27,565 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:33,097 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:27:33,098 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:27:33,314 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:34,027 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:27:38,315 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:43,895 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:44,101 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log +2023-11-22 22:27:48,097 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: stop_status +2023-11-22 22:27:48,098 DEBUG SenderThread:2219461 [sender.py:send_request():407] send_request: stop_status +2023-11-22 22:27:49,346 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:53,565 DEBUG SenderThread:2219461 [sender.py:send():380] send: stats +2023-11-22 22:27:54,895 DEBUG HandlerThread:2219461 [handler.py:handle_request():146] handle_request: status_report +2023-11-22 22:27:56,302 INFO Thread-12 :2219461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231122_100021-qrxr9o47/files/output.log