diff --git "a/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" "b/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" --- "a/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" +++ "b/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" @@ -18894,3 +18894,18573 @@ 2024-04-25 18:48:10,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status 2024-04-25 18:48:10,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status 2024-04-25 18:48:10,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:12,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:14,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:14,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:16,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:18,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:20,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:20,169 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:48:20,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:22,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:24,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:25,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:25,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:48:25,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:48:25,452 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:48:26,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:27,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:48:27,990 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:48:27,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:48:27,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:48:28,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:28,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:48:30,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:30,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:32,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:34,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:36,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:36,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:38,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:40,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:48:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:48:40,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:48:40,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:41,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:42,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:44,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:46,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:46,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:48,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:50,181 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:48:50,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:52,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:52,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:54,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:55,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:48:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:48:55,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:48:56,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:57,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:58,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:00,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:02,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:02,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:04,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:06,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:08,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:08,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:10,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:49:10,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:49:10,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:49:10,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:12,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:49:12,610 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:49:12,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:49:12,612 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:49:12,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:12,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:49:13,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:14,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:16,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:18,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:18,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:20,173 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:49:20,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:22,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:24,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:24,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:25,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:49:25,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:49:25,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:49:26,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:28,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:29,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:30,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:32,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:34,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:34,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:36,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:38,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:40,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:40,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:49:40,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:49:40,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:49:40,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:42,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:44,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:45,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:46,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:49,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:50,176 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:49:51,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:51,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:53,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:55,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:55,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:49:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:49:55,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:49:56,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:49:57,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:49:57,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:49:57,210 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:49:57,211 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:49:57,214 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:49:58,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:49:59,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:01,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:01,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:03,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:05,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:07,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:07,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:09,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:10,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:50:10,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:50:10,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:50:11,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:12,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:13,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:15,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:17,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:17,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:19,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:20,178 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:50:21,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:23,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:23,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:25,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:25,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:50:25,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:50:25,484 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:50:27,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:28,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:29,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:31,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:33,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:33,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:35,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:37,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:39,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:39,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:40,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:50:40,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:50:40,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:50:41,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:41,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:50:41,775 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:50:41,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:50:41,777 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:50:42,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:50:43,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:44,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:45,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:47,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:49,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:49,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:50,190 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:50:51,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:53,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:55,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:55,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:50:55,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:50:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:50:55,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:50:57,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:50:59,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:00,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:01,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:03,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:05,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:05,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:07,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:09,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:10,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:51:10,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:51:10,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:51:11,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:11,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:13,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:15,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:17,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:17,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:19,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:20,182 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:51:21,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:22,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:23,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:25,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:25,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:51:25,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:51:25,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:51:26,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:51:26,329 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:51:26,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:51:26,331 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:51:27,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:27,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:51:28,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:29,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:31,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:33,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:33,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:35,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:37,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:38,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:39,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:40,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:51:40,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:51:40,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:51:41,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:43,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:44,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:45,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:47,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:49,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:49,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:50,185 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:51:51,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:53,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:54,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:51:55,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:55,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:51:55,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:51:55,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:51:57,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:51:59,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:00,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:01,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:03,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:05,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:05,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:07,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:09,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:10,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:52:10,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:52:10,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:52:10,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:10,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:52:10,847 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:52:10,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:52:10,849 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:52:11,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:11,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:52:13,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:15,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:16,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:17,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:19,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:20,186 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:52:21,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:21,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:23,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:25,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:25,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:52:25,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:52:25,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:52:26,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:27,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:29,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:31,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:32,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:33,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:35,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:37,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:37,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:39,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:40,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:52:40,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:52:40,485 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:52:41,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:42,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:43,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:45,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:47,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:48,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:49,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:50,200 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:52:51,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:53,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:53,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:55,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:55,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:52:55,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:52:55,461 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:52:55,461 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:52:55,463 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:52:55,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:52:55,516 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:52:56,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:52:57,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:52:59,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:52:59,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:01,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:03,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:04,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:05,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:07,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:09,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:09,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:10,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:53:10,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:53:10,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:53:11,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:13,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:15,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:15,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:17,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:19,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:20,231 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:53:20,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:21,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:23,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:25,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:25,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:53:25,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:25,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:53:25,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:53:27,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:28,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:30,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:31,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:32,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:34,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:36,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:36,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:38,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:40,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:53:40,052 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:53:40,052 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:53:40,054 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:53:40,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:40,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:53:40,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:53:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:53:40,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:53:41,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:42,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:44,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:46,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:47,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:48,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:50,233 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:53:50,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:52,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:52,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:54,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:55,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:53:55,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:53:55,503 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:53:56,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:53:57,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:53:58,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:00,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:02,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:03,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:04,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:06,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:08,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:08,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:10,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:10,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:54:10,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:54:10,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:54:12,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:13,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:14,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:16,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:18,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:19,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:20,235 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:54:20,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:22,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:24,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:24,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:24,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:54:24,679 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:54:24,679 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:54:24,681 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:54:25,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:54:25,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:54:25,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:54:25,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:54:26,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:28,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:29,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:30,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:32,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:34,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:35,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:36,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:38,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:40,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:40,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:54:40,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:54:40,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:54:42,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:44,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:46,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:46,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:48,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:50,247 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:54:50,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:51,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:52,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:54,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:55,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:54:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:54:55,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:54:56,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:54:56,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:54:58,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:00,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:02,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:02,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:04,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:06,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:07,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:08,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:09,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:55:09,400 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:55:09,400 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:55:09,402 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:55:10,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:10,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:55:10,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:55:10,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:55:10,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:55:12,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:12,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:14,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:16,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:18,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:18,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:20,239 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:55:20,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:22,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:23,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:24,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:25,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:55:25,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:55:25,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:55:26,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:28,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:28,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:30,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:32,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:34,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:34,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:36,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:38,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:39,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:40,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:40,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:55:40,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:55:40,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:55:42,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:44,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:45,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:46,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:48,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:50,242 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:55:50,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:50,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:52,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:54,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:55:54,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:55:54,014 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:55:54,016 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:55:54,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:54,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:55:55,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:55:55,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:55:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:55:55,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:55:56,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:55:58,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:00,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:01,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:02,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:04,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:06,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:06,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:08,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:10,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:10,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:56:10,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:56:10,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:56:11,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:12,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:14,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:16,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:17,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:18,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:20,243 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:56:20,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:22,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:22,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:24,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:25,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:56:25,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:56:25,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:56:26,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:27,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:28,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:30,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:32,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:33,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:34,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:36,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:38,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:38,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:38,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:56:38,654 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:56:38,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:56:38,656 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:56:39,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:56:40,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:40,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:56:40,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:56:40,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:56:42,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:43,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:44,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:46,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:48,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:49,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:50,246 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:56:50,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:52,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:54,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:54,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:56:55,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:56:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:56:55,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:56:56,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:56:58,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:00,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:00,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:02,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:04,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:05,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:06,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:08,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:10,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:10,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:57:10,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:10,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:57:10,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:57:12,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:14,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:16,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:16,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:18,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:20,248 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:57:20,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:21,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:22,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:23,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:57:23,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:57:23,206 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:57:23,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:57:23,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:57:24,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:25,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:57:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:57:25,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:57:26,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:26,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:28,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:30,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:32,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:32,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:34,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:36,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:37,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:38,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:40,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:57:40,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:57:40,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:57:40,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:42,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:42,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:44,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:46,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:48,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:48,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:50,250 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:57:50,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:52,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:53,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:57:54,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:55,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:57:55,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:57:55,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:57:56,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:58,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:57:58,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:00,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:02,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:04,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:04,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:06,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:07,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:58:07,904 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:58:07,905 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:58:07,907 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:58:08,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:08,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:58:09,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:10,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:58:10,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:58:10,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:58:10,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:12,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:14,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:14,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:16,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:18,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:20,252 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:58:20,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:20,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:22,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:24,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:25,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:58:25,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:58:25,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:58:26,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:28,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:30,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:31,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:32,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:34,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:36,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:36,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:38,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:40,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:58:40,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:58:40,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:58:40,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:41,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:42,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:44,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:46,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:47,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:48,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:50,255 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:58:50,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:52,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:52,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:58:52,486 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:58:52,486 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:58:52,488 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:58:52,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:52,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:58:54,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:55,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:58:55,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:58:55,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:58:56,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:58:57,808 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:58:58,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:00,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:02,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:03,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:04,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:06,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:08,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:08,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:10,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:59:10,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:59:10,505 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:59:10,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:12,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:13,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:14,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:16,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:18,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:19,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:20,257 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:59:20,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:22,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:24,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:24,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:25,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:59:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:59:25,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:59:26,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:28,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:30,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:30,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:32,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:34,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:35,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:36,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:37,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:59:37,230 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:59:37,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:59:37,232 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:59:37,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:59:38,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:40,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:59:40,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:40,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:59:40,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:59:40,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:42,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:44,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:46,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:46,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:48,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:50,259 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:59:50,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:51,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:52,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:54,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:55,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:59:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:59:55,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:59:56,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:59:56,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:59:58,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:00,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:02,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:02,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:04,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:06,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:07,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:08,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:10,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:00:10,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:00:10,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:00:10,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:12,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:12,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:14,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:16,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:18,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:18,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:20,261 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:00:20,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:21,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:00:21,781 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:00:21,781 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:00:21,785 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:00:21,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:00:22,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:23,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:24,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:25,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:00:25,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:00:25,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:00:26,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:28,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:28,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:30,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:32,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:34,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:34,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:36,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:38,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:39,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:40,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:00:40,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:00:40,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:00:40,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:42,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:44,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:44,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:46,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:48,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:50,263 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:00:50,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:50,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:52,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:54,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:55,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:00:55,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:00:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:00:55,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:00:56,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:00:58,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:00,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:01,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:02,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:04,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:06,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:06,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:01:06,457 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:01:06,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:01:06,459 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:01:06,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:06,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:01:08,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:10,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:01:10,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:01:10,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:01:10,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:11,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:12,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:14,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:16,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:17,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:18,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:20,266 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:01:21,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:22,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:23,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:25,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:25,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:01:25,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:01:25,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:01:27,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:27,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:29,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:31,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:33,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:33,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:35,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:37,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:38,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:39,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:40,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:01:40,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:01:40,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:01:41,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:43,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:43,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:45,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:47,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:49,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:49,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:50,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:01:51,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:01:51,008 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:01:51,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:01:51,010 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:01:51,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:51,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:01:53,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:54,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:01:55,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:55,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:01:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:01:55,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:01:57,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:59,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:01:59,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:01,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:03,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:05,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:05,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:07,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:09,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:10,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:02:10,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:02:10,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:02:11,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:13,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:15,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:15,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:17,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:19,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:20,270 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:02:21,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:21,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:23,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:25,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:25,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:02:25,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:02:25,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:02:26,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:27,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:29,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:31,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:31,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:33,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:35,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:35,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:02:35,589 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:02:35,589 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:02:35,591 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:02:36,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:02:37,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:37,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:39,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:40,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:02:40,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:02:40,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:02:41,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:42,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:43,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:45,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:47,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:48,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:49,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:50,272 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:02:51,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:53,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:53,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:55,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:55,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:02:55,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:02:55,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:02:57,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:02:58,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:02:59,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:01,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:03,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:04,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:05,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:07,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:09,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:09,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:10,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:03:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:03:10,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:03:11,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:13,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:14,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:15,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:17,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:19,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:20,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:20,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:03:20,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:03:20,277 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:03:20,277 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:03:20,279 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:03:21,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:21,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:03:23,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:25,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:25,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:03:25,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:25,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:03:25,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:03:30,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:31,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:33,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:35,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:36,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:37,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:39,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:40,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:03:40,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:03:40,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:03:41,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:41,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:43,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:45,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:46,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:47,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:49,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:50,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:03:51,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:52,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:53,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:55,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:55,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:03:55,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:03:55,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:03:57,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:03:57,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:03:59,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:01,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:03,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:03,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:04,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:04:04,989 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:04:04,989 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:04:04,991 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:04:05,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:05,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:04:07,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:08,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:09,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:10,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:04:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:04:10,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:04:11,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:13,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:13,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:15,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:17,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:19,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:19,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:20,279 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:04:21,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:23,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:24,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:25,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:25,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:04:25,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:04:25,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:04:27,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:29,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:29,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:31,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:33,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:35,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:35,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:37,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:39,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:40,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:04:40,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:40,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:04:40,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:04:41,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:43,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:45,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:45,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:47,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:49,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:49,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:04:49,507 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:04:49,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:04:49,509 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:04:50,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:04:50,281 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:04:51,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:51,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:53,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:55,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:55,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:04:55,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:04:55,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:04:56,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:04:57,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:04:59,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:01,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:02,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:03,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:05,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:07,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:07,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:09,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:10,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:05:10,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:05:10,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:05:11,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:12,770 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:13,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:15,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:17,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:18,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:19,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:20,283 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:05:21,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:23,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:23,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:25,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:25,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:05:25,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:05:25,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:05:27,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:28,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:29,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:31,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:33,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:34,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:34,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:05:34,182 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:05:34,183 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:05:34,185 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:05:34,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:05:35,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:37,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:39,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:39,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:40,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:05:40,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:05:40,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:05:41,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:43,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:44,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:45,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:47,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:49,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:50,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:50,285 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:05:51,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:53,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:55,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:55,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:05:55,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:05:55,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:05:55,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:05:57,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:05:59,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:00,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:01,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:03,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:05,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:06,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:07,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:09,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:10,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:06:10,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:06:10,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:06:11,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:11,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:13,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:15,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:17,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:17,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:18,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:06:18,843 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:06:18,843 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:06:18,845 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:06:19,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:19,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:06:20,287 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:06:21,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:22,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:23,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:25,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:25,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:06:25,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:06:25,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:06:27,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:27,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:29,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:31,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:33,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:33,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:35,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:37,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:38,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:39,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:40,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:06:40,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:06:40,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:06:41,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:43,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:43,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:45,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:47,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:49,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:49,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:50,289 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:06:51,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:53,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:54,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:06:55,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:55,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:06:55,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:06:55,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:06:57,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:59,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:06:59,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:01,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:03,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:03,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:07:03,380 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:07:03,380 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:07:03,382 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:07:04,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:07:05,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:05,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:07,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:09,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:10,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:07:10,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:10,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:07:10,549 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:07:11,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:13,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:15,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:15,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:17,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:19,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:20,291 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:07:21,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:21,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:23,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:25,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:25,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:07:25,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:07:25,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:07:26,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:27,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:29,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:31,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:31,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:33,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:35,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:37,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:37,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:39,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:40,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:07:40,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:07:40,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:07:41,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:42,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:43,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:45,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:47,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:48,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:48,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:07:48,038 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:07:48,038 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:07:48,040 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:07:48,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:07:49,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:50,293 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:07:51,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:53,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:53,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:55,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:55,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:07:55,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:07:55,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:07:57,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:07:58,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:07:59,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:01,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:03,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:04,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:05,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:07,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:09,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:09,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:10,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:08:10,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:08:10,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:08:10,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:12,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:14,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:14,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:16,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:18,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:20,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:20,296 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:08:20,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:22,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:24,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:25,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:08:25,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:25,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:08:25,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:08:26,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:28,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:30,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:30,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:32,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:32,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:08:32,714 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:08:32,714 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:08:32,716 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:08:33,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:08:34,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:36,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:36,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:38,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:40,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:08:40,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:08:40,509 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:08:40,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:41,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:42,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:44,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:46,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:46,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:48,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:50,297 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:08:50,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:52,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:52,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:54,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:55,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:08:55,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:08:55,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:08:56,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:08:57,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:08:58,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:00,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:02,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:02,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:04,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:06,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:08,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:08,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:10,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:09:10,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:09:10,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:09:10,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:12,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:13,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:14,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:16,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:17,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:09:17,274 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:09:17,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:09:17,276 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:09:17,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:09:18,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:19,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:20,299 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:09:20,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:22,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:24,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:24,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:25,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:09:25,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:09:25,510 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:09:26,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:28,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:29,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:30,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:32,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:34,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:35,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:36,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:38,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:40,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:09:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:09:40,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:09:40,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:42,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:44,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:45,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:46,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:48,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:50,301 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:09:50,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:51,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:52,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:54,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:55,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:09:55,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:09:55,526 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:09:56,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:09:56,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:09:58,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:00,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:01,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:01,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:10:01,876 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:10:01,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:10:01,878 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:10:02,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:02,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:10:04,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:06,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:07,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:08,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:10,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:10:10,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:10:10,526 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:10:10,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:12,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:12,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:14,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:16,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:17,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:18,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:20,304 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:10:20,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:22,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:23,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:24,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:25,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:10:25,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:10:25,525 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:10:26,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:28,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:28,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:30,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:32,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:34,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:34,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:36,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:38,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:39,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:40,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:10:40,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:10:40,525 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:10:40,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:42,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:44,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:44,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:46,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:10:46,567 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:10:46,567 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:10:46,570 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:10:46,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:46,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:10:48,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:50,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:50,306 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:10:50,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:52,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:54,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:55,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:10:55,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:10:55,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:10:55,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:10:56,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:10:58,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:00,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:00,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:02,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:04,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:06,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:06,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:08,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:11:10,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:11:10,539 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:11:10,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:11,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:12,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:14,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:16,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:16,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:18,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:20,308 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:11:20,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:22,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:22,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:24,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:25,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:11:25,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:11:25,539 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:11:26,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:27,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:28,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:30,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:31,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:11:31,155 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:11:31,156 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:11:31,158 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:11:31,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:11:32,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:32,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:34,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:36,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:38,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:38,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:40,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:11:40,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:11:40,539 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:11:40,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:42,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:43,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:44,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:46,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:48,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:48,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:50,310 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:11:50,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:52,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:54,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:11:54,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:55,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:11:55,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:11:55,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:11:56,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:58,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:11:59,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:00,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:02,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:04,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:04,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:06,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:08,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:10,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:12:10,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:12:10,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:12:10,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:12,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:14,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:15,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:15,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:12:15,733 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:12:15,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:12:15,735 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:12:15,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:12:16,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:18,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:20,313 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:12:20,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:21,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:22,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:24,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:25,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:12:25,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:12:25,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:12:26,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:26,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:28,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:30,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:31,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:32,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:34,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:36,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:37,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:38,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:40,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:12:40,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:12:40,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:12:40,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:42,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:42,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:44,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:46,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:47,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:48,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:50,314 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:12:50,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:52,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:53,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:55,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:55,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:12:55,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:12:55,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:12:57,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:12:58,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:12:59,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:00,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:13:00,273 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:13:00,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:13:00,275 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:13:01,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:01,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:13:03,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:03,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:05,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:07,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:09,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:09,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:13:10,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:13:10,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:13:11,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:13,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:14,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:15,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:17,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:19,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:19,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:20,317 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:13:21,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:23,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:25,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:25,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:25,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:13:25,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:13:25,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:13:27,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:29,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:30,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:31,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:33,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:35,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:35,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:37,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:39,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:40,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:13:40,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:13:40,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:13:41,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:41,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:43,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:44,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:13:44,800 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:13:44,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:13:44,802 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:13:45,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:45,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:13:46,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:47,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:49,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:50,319 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:13:51,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:51,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:53,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:55,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:13:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:13:55,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:13:57,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:13:57,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:13:59,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:01,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:02,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:03,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:05,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:07,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:07,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:09,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:14:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:14:10,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:14:11,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:13,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:13,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:15,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:17,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:18,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:19,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:20,321 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:14:21,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:23,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:24,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:25,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:14:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:14:25,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:14:27,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:29,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:29,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:29,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:14:29,412 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:14:29,412 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:14:29,414 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:14:30,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:14:31,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:33,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:34,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:35,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:37,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:39,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:40,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:40,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:14:40,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:14:40,543 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:14:41,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:43,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:45,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:45,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:47,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:49,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:50,323 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:14:50,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:51,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:53,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:55,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:14:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:14:55,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:14:56,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:14:57,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:14:59,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:01,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:01,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:03,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:05,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:06,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:07,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:09,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:15:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:15:10,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:15:11,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:12,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:13,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:14,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:15:14,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:15:14,015 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:15:14,017 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:15:14,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:15:15,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:17,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:17,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:19,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:20,325 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:15:21,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:22,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:23,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:25,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:15:25,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:15:25,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:15:27,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:28,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:29,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:31,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:33,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:33,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:35,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:37,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:38,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:39,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:40,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:15:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:15:40,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:15:41,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:43,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:44,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:45,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:47,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:49,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:49,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:50,327 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:15:51,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:53,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:54,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:15:55,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:15:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:15:55,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:15:57,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:58,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:15:58,563 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:15:58,563 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:15:58,565 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:15:59,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:15:59,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:16:00,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:01,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:03,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:05,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:05,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:07,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:09,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:10,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:16:10,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:16:10,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:16:10,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:11,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:13,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:15,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:16,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:17,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:19,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:20,329 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:16:21,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:21,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:23,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:25,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:25,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:16:25,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:16:25,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:16:26,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:27,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:29,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:31,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:32,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:33,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:35,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:37,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:37,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:39,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:16:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:16:40,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:16:41,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:43,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:43,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:16:43,094 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:16:43,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:16:43,097 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:16:43,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:43,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:16:45,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:47,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:48,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:49,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:50,332 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:16:51,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:53,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:53,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:55,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:16:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:16:55,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:16:57,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:16:59,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:16:59,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:01,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:03,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:04,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:05,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:07,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:09,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:09,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:10,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:17:10,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:17:10,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:17:11,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:13,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:15,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:15,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:17,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:19,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:20,411 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:17:20,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:21,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:23,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:25,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:25,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:17:25,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:25,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:17:25,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:17:27,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:27,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:17:27,702 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:17:27,702 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:17:27,704 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:17:28,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:17:29,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:31,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:31,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:33,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:35,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:36,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:37,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:39,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:17:40,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:17:40,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:17:41,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:41,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:43,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:45,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:47,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:47,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:49,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:50,339 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:17:51,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:52,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:53,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:55,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:55,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:17:55,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:17:55,545 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:17:57,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:17:57,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:17:59,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:01,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:03,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:03,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:05,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:07,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:08,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:09,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:18:10,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:18:10,582 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:18:11,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:12,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:18:12,327 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:18:12,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:18:12,329 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:18:12,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:18:13,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:14,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:15,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:17,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:19,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:19,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:20,341 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:18:21,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:23,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:24,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:25,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:25,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:18:25,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:18:25,579 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:18:27,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:29,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:30,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:31,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:33,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:35,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:35,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:37,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:39,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:40,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:18:40,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:40,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:18:40,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:18:41,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:43,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:45,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:46,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:47,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:49,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:50,343 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:18:51,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:51,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:53,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:55,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:18:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:18:55,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:18:55,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:56,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:18:56,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:18:56,935 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:18:56,936 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:18:56,937 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:18:57,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:18:57,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:18:59,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:01,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:02,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:03,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:05,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:07,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:07,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:09,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:10,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:19:10,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:19:10,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:19:11,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:12,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:13,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:15,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:17,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:18,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:19,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:20,345 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:19:21,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:23,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:23,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:25,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:19:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:19:25,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:19:25,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:27,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:28,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:29,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:31,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:33,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:34,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:35,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:37,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:39,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:39,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:40,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:19:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:19:40,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:19:41,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:19:41,519 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:19:41,519 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:19:41,521 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:19:41,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:41,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:19:43,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:45,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:47,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:49,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:50,347 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:19:50,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:51,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:53,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:55,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:19:55,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:19:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:19:55,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:19:55,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:57,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:19:59,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:01,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:01,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:03,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:05,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:06,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:07,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:09,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:10,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:20:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:20:10,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:20:11,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:11,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:13,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:15,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:17,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:17,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:19,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:20,350 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:20:21,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:22,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:23,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:25,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:20:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:20:25,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:20:25,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:26,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:20:26,095 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:20:26,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:20:26,097 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:20:26,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:20:27,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:27,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:29,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:31,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:33,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:33,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:34,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:36,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:38,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:38,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:40,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:20:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:20:40,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:20:40,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:42,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:43,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:44,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:46,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:48,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:49,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:50,351 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:20:50,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:52,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:54,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:20:54,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:55,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:20:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:20:55,580 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:20:56,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:58,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:20:59,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:00,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:02,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:04,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:05,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:06,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:08,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:10,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:21:10,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:21:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:21:10,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:21:10,657 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:21:10,658 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:21:10,661 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:21:10,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:10,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:21:12,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:14,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:15,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:16,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:18,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:20,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:21:20,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:21,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:22,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:24,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:25,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:21:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:21:25,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:21:26,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:26,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:28,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:30,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:31,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:32,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:34,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:36,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:37,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:38,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:40,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:21:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:21:40,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:21:40,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:42,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:42,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:44,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:46,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:48,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:48,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:50,355 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:21:50,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:52,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:53,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:54,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:55,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:21:55,193 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:21:55,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:21:55,195 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:21:55,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:21:55,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:21:55,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:21:55,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:21:56,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:21:58,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:21:58,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:00,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:02,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:04,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:04,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:06,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:08,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:09,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:10,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:22:10,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:22:10,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:22:10,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:12,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:14,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:14,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:16,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:18,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:20,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:20,357 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:22:20,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:22,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:24,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:25,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:25,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:22:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:22:25,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:22:26,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:28,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:30,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:30,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:32,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:34,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:36,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:36,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:38,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:39,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:22:39,821 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:22:39,821 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:22:39,823 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:22:39,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:22:40,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:22:40,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:22:40,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:22:40,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:41,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:42,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:44,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:46,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:46,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:48,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:50,360 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:22:50,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:52,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:52,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:54,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:55,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:22:55,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:22:55,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:22:56,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:22:57,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:22:58,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:00,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:02,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:03,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:04,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:06,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:08,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:08,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:10,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:23:10,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:23:10,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:23:10,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:12,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:13,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:14,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:16,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:18,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:19,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:20,373 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:23:20,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:22,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:24,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:24,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:23:24,435 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:23:24,435 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:23:24,437 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:23:24,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:24,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:23:25,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:23:25,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:23:25,583 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:23:26,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:28,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:29,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:30,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:32,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:34,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:35,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:36,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:38,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:40,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:40,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:23:40,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:23:40,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:23:40,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:42,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:44,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:45,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:46,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:48,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:50,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:23:50,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:51,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:52,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:54,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:55,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:23:55,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:23:55,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:23:56,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:23:56,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:23:58,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:00,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:01,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:02,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:04,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:06,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:07,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:08,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:09,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:24:09,057 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:24:09,057 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:24:09,059 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:24:09,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:24:10,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:24:10,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:24:10,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:24:10,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:12,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:12,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:14,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:16,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:17,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:18,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:20,366 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:24:20,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:22,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:23,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:25,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:25,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:24:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:24:25,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:24:27,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:28,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:29,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:31,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:33,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:34,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:35,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:37,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:39,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:39,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:40,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:24:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:24:40,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:24:41,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:43,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:44,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:45,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:47,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:49,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:50,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:50,368 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:24:51,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:53,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:53,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:24:53,742 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:24:53,742 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:24:53,744 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:24:54,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:24:55,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:55,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:24:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:24:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:24:55,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:24:57,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:24:59,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:00,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:01,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:03,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:05,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:06,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:07,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:09,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:10,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:25:10,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:25:10,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:25:11,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:11,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:13,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:15,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:16,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:17,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:20,381 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:25:21,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:22,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:23,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:25,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:25,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:25:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:25:25,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:25:27,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:27,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:29,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:31,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:32,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:33,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:35,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:37,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:38,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:38,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:25:38,292 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:25:38,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:25:38,294 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:25:39,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:39,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:25:40,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:25:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:25:40,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:25:41,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:43,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:43,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:45,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:47,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:48,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:49,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:50,373 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:25:51,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:53,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:54,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:25:55,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:25:55,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:25:55,584 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:25:57,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:59,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:25:59,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:01,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:03,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:05,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:05,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:07,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:09,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:10,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:10,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:26:10,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:26:10,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:26:11,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:13,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:15,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:15,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:17,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:19,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:20,375 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:26:21,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:22,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:26:22,861 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:26:22,861 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:26:22,863 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:26:23,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:23,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:26:25,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:25,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:26:25,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:26:25,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:26:26,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:27,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:29,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:31,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:31,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:33,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:35,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:37,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:37,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:39,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:40,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:26:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:26:40,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:26:41,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:42,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:43,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:45,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:47,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:47,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:49,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:50,377 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:26:51,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:53,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:53,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:55,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:55,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:26:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:26:55,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:26:57,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:26:58,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:26:59,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:01,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:03,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:03,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:05,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:07,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:07,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:27:07,325 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:27:07,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:27:07,327 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:27:08,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:27:09,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:09,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:27:10,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:27:10,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:27:11,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:13,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:14,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:15,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:17,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:19,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:19,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:20,391 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:27:21,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:23,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:25,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:25,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:27:25,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:27:25,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:27:27,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:29,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:30,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:31,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:33,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:35,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:35,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:37,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:39,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:27:40,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:27:40,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:27:41,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:41,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:43,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:45,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:46,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:47,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:49,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:50,381 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:27:51,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:51,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:51,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:27:51,929 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:27:51,929 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:27:51,931 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:27:52,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:27:53,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:55,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:27:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:27:55,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:27:57,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:27:57,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:27:59,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:01,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:02,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:03,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:05,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:07,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:07,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:09,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:10,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:28:10,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:28:10,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:28:11,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:13,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:13,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:15,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:17,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:18,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:19,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:20,383 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:28:21,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:23,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:24,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:25,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:25,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:28:25,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:28:25,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:28:27,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:29,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:29,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:31,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:33,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:34,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:35,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:36,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:28:36,569 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:28:36,570 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:28:36,571 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:28:37,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:37,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:28:39,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:40,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:40,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:28:40,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:28:40,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:28:41,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:43,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:45,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:45,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:47,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:49,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:50,386 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:28:50,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:51,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:53,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:55,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:28:55,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:28:55,586 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:28:56,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:28:57,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:28:59,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:01,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:01,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:03,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:05,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:06,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:07,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:09,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:10,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:29:10,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:29:10,618 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:29:11,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:12,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:13,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:15,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:17,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:17,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:19,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:20,405 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:29:21,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:29:21,256 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:29:21,256 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:29:21,258 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:29:21,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:21,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:29:23,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:23,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:25,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:25,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:29:25,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:29:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:29:27,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:28,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:29,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:31,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:33,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:33,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:35,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:37,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:39,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:39,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:40,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:29:40,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:29:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:29:41,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:43,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:44,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:45,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:47,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:49,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:49,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:50,390 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:29:51,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:53,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:55,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:29:55,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:29:55,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:29:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:29:57,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:29:59,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:00,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:01,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:03,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:05,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:05,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:05,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:30:05,860 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:30:05,861 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:30:05,862 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:30:06,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:30:07,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:09,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:30:10,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:30:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:30:11,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:11,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:13,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:15,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:16,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:17,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:19,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:20,392 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:30:21,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:21,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:23,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:25,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:30:25,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:30:25,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:30:27,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:27,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:29,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:31,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:32,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:33,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:35,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:37,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:39,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:40,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:30:40,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:30:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:30:41,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:43,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:43,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:45,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:47,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:48,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:49,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:50,394 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:30:50,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:30:50,445 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:30:50,445 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:30:50,447 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:30:50,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:30:51,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:53,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:53,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:30:55,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:30:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:30:55,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:57,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:30:59,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:30:59,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:01,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:03,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:04,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:05,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:07,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:09,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:10,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:31:10,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:31:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:31:11,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:13,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:15,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:15,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:17,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:19,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:20,409 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:31:20,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:21,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:23,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:25,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:25,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:31:25,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:31:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:31:25,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:27,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:29,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:30,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:31,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:35,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:31:35,072 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:31:35,072 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:31:35,075 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:31:35,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:31:36,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:37,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:39,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:40,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:31:40,610 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:31:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:31:41,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:41,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:43,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:45,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:46,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:47,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:49,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:50,399 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:31:51,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:52,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:53,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:31:55,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:31:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:31:55,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:57,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:31:57,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:31:59,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:01,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:02,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:03,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:05,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:07,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:08,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:09,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:10,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:32:10,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:32:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:32:11,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:13,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:13,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:15,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:17,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:18,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:19,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:32:19,623 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:32:19,623 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:32:19,625 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:32:19,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:19,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:32:20,401 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:32:21,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:23,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:24,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:25,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:32:25,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:32:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:32:25,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:27,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:29,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:29,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:31,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:33,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:34,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:35,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:37,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:39,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:40,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:40,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:32:40,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:32:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:32:41,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:43,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:45,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:45,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:47,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:49,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:50,403 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:32:51,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:51,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:53,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:55,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:32:55,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:32:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:32:55,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:56,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:32:57,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:32:59,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:01,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:02,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:03,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:04,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:33:04,116 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:33:04,117 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:33:04,119 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:33:04,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:33:05,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:07,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:07,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:09,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:10,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:33:10,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:33:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:33:11,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:12,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:13,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:15,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:17,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:18,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:19,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:20,419 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:33:21,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:23,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:23,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:25,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:33:25,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:33:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:33:25,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:27,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:28,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:29,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:31,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:33,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:34,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:35,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:37,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:39,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:39,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:40,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:33:40,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:33:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:33:41,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:43,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:45,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:45,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:47,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:48,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:33:48,682 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:33:48,682 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:33:48,684 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:33:48,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:33:49,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:50,407 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:33:50,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:51,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:53,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:55,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:33:55,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:33:55,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:33:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:33:55,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:57,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:33:59,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:01,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:01,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:03,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:05,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:06,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:07,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:09,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:10,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:34:10,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:34:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:34:11,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:11,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:13,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:15,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:17,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:17,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:19,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:20,409 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:34:21,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:22,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:23,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:25,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:34:25,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:34:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:34:25,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:27,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:27,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:29,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:31,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:33,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:33,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:34:33,272 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:34:33,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:34:33,274 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:34:33,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:33,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:34:35,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:37,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:38,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:39,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:40,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:34:40,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:34:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:34:41,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:43,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:43,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:45,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:47,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:49,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:49,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:50,411 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:34:51,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:53,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:54,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:34:55,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:34:55,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:34:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:34:55,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:57,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:34:59,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:00,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:01,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:03,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:05,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:05,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:07,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:09,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:10,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:10,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:35:10,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:35:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:35:11,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:13,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:15,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:16,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:17,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:35:17,924 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:35:17,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:35:17,926 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:35:17,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:17,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:35:19,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:20,425 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:35:21,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:21,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:23,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:25,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:35:25,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:35:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:35:25,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:26,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:27,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:29,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:31,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:32,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:33,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:35,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:37,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:37,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:39,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:40,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:35:40,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:35:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:35:41,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:42,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:43,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:45,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:47,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:48,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:49,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:50,415 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:35:51,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:53,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:53,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:55,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:35:55,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:35:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:35:55,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:57,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:35:58,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:35:59,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:01,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:02,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:36:02,478 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:36:02,479 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:36:02,481 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:36:02,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:36:03,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:04,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:05,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:07,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:09,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:09,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:10,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:36:10,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:36:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:36:11,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:13,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:14,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:15,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:18,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:20,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:20,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:20,417 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:36:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:25,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:25,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:36:25,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:36:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:36:26,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:28,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:30,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:31,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:32,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:34,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:36,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:36,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:38,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:40,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:40,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:36:40,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:36:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:36:41,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:42,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:44,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:46,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:47,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:47,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:36:47,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:36:47,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:36:47,209 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:36:48,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:48,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:36:50,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:50,420 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:36:52,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:52,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:54,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:55,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:36:55,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:36:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:36:56,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:36:57,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:36:58,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:00,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:02,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:03,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:04,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:06,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:08,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:08,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:10,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:10,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:37:10,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:37:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:37:12,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:13,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:14,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:16,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:18,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:19,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:20,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:20,434 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:37:22,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:24,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:24,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:25,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:37:25,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:37:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:37:26,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:28,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:30,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:30,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:31,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:37:31,858 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:37:31,859 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:37:31,860 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:37:32,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:32,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:37:33,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:35,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:36,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:38,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:40,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:40,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:40,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:37:40,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:37:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:37:42,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:44,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:46,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:46,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:48,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:50,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:50,425 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:37:51,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:52,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:54,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:55,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:55,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:37:55,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:37:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:37:56,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:37:57,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:37:59,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:01,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:02,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:03,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:05,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:07,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:07,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:09,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:10,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:38:10,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:38:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:38:11,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:12,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:13,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:15,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:16,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:38:16,482 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:38:16,483 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:38:16,485 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:38:17,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:17,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:38:18,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:19,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:20,427 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:38:21,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:23,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:23,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:25,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:25,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:38:25,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:38:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:38:27,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:28,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:29,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:31,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:33,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:34,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:35,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:37,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:39,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:39,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:40,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:38:40,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:38:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:38:41,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:45,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:45,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:47,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:49,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:50,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:50,429 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:38:51,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:53,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:55,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:55,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:38:55,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:38:55,613 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:38:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:38:57,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:38:59,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:01,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:01,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:39:01,076 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:39:01,076 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:39:01,078 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:39:01,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:01,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:39:03,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:05,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:06,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:07,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:09,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:10,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:39:10,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:39:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:39:11,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:11,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:13,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:15,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:17,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:17,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:19,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:20,447 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:39:21,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:22,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:23,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:25,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:25,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:39:25,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:39:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:39:27,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:27,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:29,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:31,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:33,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:33,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:35,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:37,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:38,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:39,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:40,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:39:40,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:39:40,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:39:41,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:43,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:43,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:45,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:45,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:39:45,710 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:39:45,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:39:45,712 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:39:46,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:39:47,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:49,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:49,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:50,433 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:39:51,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:53,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:54,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:39:55,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:55,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:39:55,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:39:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:39:57,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:59,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:39:59,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:01,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:03,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:05,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:05,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:07,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:09,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:10,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:10,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:40:10,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:40:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:40:11,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:13,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:15,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:16,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:17,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:19,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:20,436 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:40:21,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:21,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:23,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:25,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:25,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:40:25,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:40:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:40:26,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:27,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:29,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:30,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:40:30,348 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:40:30,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:40:30,350 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:40:30,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:40:31,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:32,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:33,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:35,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:37,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:37,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:39,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:40,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:40:40,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:40:40,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:40:41,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:42,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:43,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:45,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:47,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:48,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:49,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:50,438 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:40:51,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:53,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:53,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:55,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:55,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:40:55,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:40:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:40:57,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:40:58,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:40:59,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:01,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:03,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:04,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:05,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:07,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:09,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:09,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:10,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:41:10,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:41:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:41:11,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:13,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:14,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:14,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:41:14,956 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:41:14,957 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:41:14,958 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:41:15,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:15,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:41:17,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:19,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:20,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:20,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:41:21,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:23,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:25,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:25,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:25,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:41:25,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:41:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:41:27,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:29,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:31,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:31,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:33,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:35,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:36,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:37,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:39,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:40,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:41:40,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:41:40,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:41:41,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:41,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:43,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:45,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:47,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:47,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:49,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:50,442 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:41:51,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:52,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:53,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:55,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:55,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:41:55,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:41:55,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:41:57,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:57,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:41:59,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:41:59,540 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:41:59,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:41:59,543 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:41:59,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:41:59,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:42:01,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:03,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:03,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:05,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:07,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:08,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:09,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:10,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:42:10,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:42:10,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:42:11,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:13,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:13,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:15,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:17,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:19,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:19,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:20,444 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:42:21,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:23,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:24,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:25,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:42:25,615 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:42:25,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:25,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:42:27,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:29,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:29,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:31,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:33,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:35,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:35,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:37,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:39,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:40,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:40,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:42:40,663 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:42:40,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:42:41,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:43,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:44,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:42:44,179 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:42:44,180 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:42:44,182 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:42:44,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:42:45,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:45,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:47,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:49,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:50,446 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:42:51,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:51,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:53,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:55,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:42:55,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:42:55,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:42:56,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:42:57,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:42:59,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:01,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:02,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:03,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:05,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:07,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:07,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:09,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:10,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:43:10,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:43:10,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:43:11,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:12,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:13,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:15,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:17,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:18,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:19,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:20,459 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:43:21,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:23,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:23,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:25,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:43:25,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:43:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:43:25,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:27,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:28,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:28,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:43:28,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:43:28,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:43:28,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:43:29,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:29,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:43:31,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:33,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:34,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:35,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:39,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:40,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:43:40,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:43:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:43:41,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:43,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:44,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:45,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:47,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:49,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:50,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:50,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:43:51,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:53,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:55,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:43:55,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:43:55,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:43:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:43:55,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:57,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:43:59,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:00,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:01,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:03,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:05,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:06,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:07,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:09,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:10,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:44:10,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:44:10,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:44:11,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:11,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:13,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:44:13,402 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:44:13,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:44:13,404 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:44:13,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:13,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:44:15,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:16,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:17,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:19,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:20,453 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:44:21,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:22,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:23,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:25,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:44:25,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:44:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:44:25,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:27,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:27,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:29,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:31,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:33,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:33,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:35,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:37,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:38,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:39,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:40,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:44:40,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:44:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:44:41,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:43,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:43,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:45,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:47,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:49,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:49,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:50,455 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:44:51,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:53,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:54,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:55,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:44:55,644 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:44:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:44:55,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:57,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:44:58,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:44:58,037 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:44:58,038 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:44:58,039 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:44:58,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:44:59,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:44:59,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:01,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:03,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:05,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:05,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:07,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:09,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:10,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:10,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:45:10,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:45:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:45:11,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:13,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:15,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:15,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:17,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:19,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:20,469 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:45:21,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:21,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:23,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:25,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:45:25,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:45:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:45:25,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:26,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:27,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:29,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:31,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:31,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:33,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:35,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:37,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:37,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:39,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:40,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:45:40,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:45:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:45:41,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:42,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:42,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:45:42,587 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:45:42,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:45:42,589 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:45:42,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:45:43,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:45,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:47,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:47,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:49,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:50,459 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:45:51,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:53,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:53,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:55,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:45:55,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:45:55,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:45:55,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:57,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:45:58,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:45:59,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:01,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:03,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:03,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:05,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:07,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:09,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:09,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:10,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:46:10,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:46:10,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:46:11,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:13,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:14,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:15,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:17,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:19,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:19,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:20,461 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:46:21,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:23,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:25,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:25,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:46:25,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:46:25,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:46:25,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:27,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:46:27,135 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:46:27,136 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:46:27,137 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:46:27,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:27,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:46:29,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:30,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:31,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:33,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:35,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:35,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:37,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:39,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:40,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:46:40,645 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:46:40,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:46:41,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:41,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:43,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:45,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:46,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:47,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:49,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:50,463 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:46:51,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:51,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:53,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:55,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:46:55,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:46:55,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:46:55,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:57,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:46:57,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:46:59,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:01,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:02,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:03,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:05,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:07,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:08,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:09,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:10,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:47:10,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:47:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:47:11,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:47:11,683 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:47:11,684 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:47:11,685 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:47:11,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:11,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:47:13,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:13,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:15,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:17,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:18,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:19,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:20,465 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:47:21,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:23,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:24,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:25,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:47:25,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:47:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:47:25,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:27,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:29,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:29,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:31,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:33,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:34,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:35,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:37,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:39,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:40,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:40,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:47:40,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:47:40,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:47:41,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:43,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:45,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:45,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:47,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:49,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:50,468 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:47:50,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:51,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:53,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:55,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:47:55,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:47:55,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:47:55,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:56,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:47:56,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:47:56,255 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:47:56,256 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:47:56,257 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:47:56,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:47:57,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:47:59,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:01,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:02,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:04,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:06,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:06,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:08,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:10,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:10,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:48:10,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:48:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:48:12,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:12,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:14,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:16,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:17,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:20,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:20,470 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:48:22,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:22,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:24,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:25,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:48:25,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:48:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:48:26,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:28,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:28,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:30,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:32,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:33,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:34,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:36,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:38,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:38,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:40,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:40,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:48:40,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:48:40,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:48:40,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:48:40,766 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:48:40,766 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:48:40,767 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:48:41,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:41,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:48:44,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:44,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:46,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:48,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:49,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:50,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:50,472 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:48:52,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:54,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:54,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:48:55,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:48:55,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:48:55,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:48:56,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:48:58,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:00,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:00,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:02,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:04,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:05,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:06,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:08,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:10,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:10,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:10,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:49:10,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:49:10,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:49:12,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:14,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:15,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:16,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:17,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:19,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:20,474 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:49:21,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:21,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:23,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:25,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:49:25,258 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:49:25,259 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:49:25,261 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:49:25,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:25,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:49:25,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:49:25,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:49:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:49:26,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:27,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:29,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:31,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:32,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:33,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:35,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:37,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:39,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:40,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:49:40,647 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:49:40,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:49:41,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:43,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:43,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:45,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:47,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:48,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:49,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:50,476 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:49:51,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:53,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:53,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:55,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:55,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:49:55,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:49:55,666 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:49:57,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:49:59,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:49:59,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:01,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:03,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:04,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:05,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:07,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:09,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:09,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:09,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:50:09,869 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:50:09,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:50:09,871 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:50:10,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:50:10,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:50:10,651 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:50:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:50:11,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:13,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:15,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:15,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:17,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:19,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:20,478 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:50:20,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:21,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:23,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:25,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:25,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:25,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:50:25,651 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:50:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:50:27,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:29,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:31,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:31,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:33,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:35,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:36,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:37,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:39,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:40,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:50:40,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:50:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:50:41,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:41,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:43,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:45,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:47,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:47,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:49,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:50,480 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:50:51,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:53,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:54,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:50:54,512 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:50:54,513 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:50:54,515 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:50:55,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:55,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:50:55,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:50:55,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:50:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:50:57,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:50:58,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:50:59,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:01,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:03,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:03,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:05,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:07,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:08,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:09,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:10,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:51:10,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:51:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:51:11,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:13,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:14,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:15,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:17,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:19,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:19,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:20,483 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:51:21,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:23,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:24,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:25,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:25,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:51:25,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:51:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:51:27,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:29,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:30,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:31,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:33,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:35,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:35,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:37,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:39,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:51:39,219 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:51:39,219 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:51:39,221 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:51:39,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:39,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:51:40,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:40,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:51:40,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:51:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:51:41,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:43,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:45,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:46,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:47,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:49,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:50,485 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:51:51,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:51,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:53,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:55,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:55,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:51:55,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:51:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:51:56,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:51:57,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:51:59,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:01,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:02,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:03,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:05,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:07,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:07,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:09,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:10,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:52:10,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:52:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:52:11,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:13,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:13,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:15,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:17,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:18,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:19,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:20,487 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:52:21,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:23,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:23,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:23,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:52:23,718 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:52:23,718 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:52:23,721 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:52:24,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:52:25,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:25,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:52:25,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:52:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:52:27,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:29,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:29,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:31,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:33,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:34,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:35,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:37,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:39,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:39,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:40,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:52:40,652 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:52:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:52:41,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:43,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:44,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:45,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:47,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:49,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:50,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:50,489 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:52:51,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:53,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:55,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:55,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:52:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:52:55,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:52:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:52:57,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:52:59,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:00,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:01,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:03,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:05,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:06,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:07,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:08,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:53:08,160 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:53:08,161 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:53:08,163 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:53:08,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:53:09,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:10,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:53:10,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:53:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:53:11,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:11,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:13,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:15,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:17,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:17,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:19,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:20,492 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:53:21,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:22,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:23,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:25,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:53:25,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:53:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:53:27,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:27,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:29,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:31,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:33,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:33,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:35,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:37,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:38,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:39,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:53:40,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:53:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:53:41,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:43,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:43,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:45,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:47,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:49,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:49,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:50,493 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:53:51,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:52,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:53:52,659 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:53:52,659 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:53:52,661 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:53:53,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:53,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:53:54,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:53:55,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:53:55,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:53:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:53:57,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:59,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:53:59,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:01,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:03,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:05,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:05,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:07,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:09,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:10,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:54:10,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:54:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:54:11,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:13,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:15,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:15,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:17,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:19,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:20,496 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:54:21,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:21,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:23,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:54:25,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:54:25,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:54:26,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:27,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:29,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:31,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:31,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:33,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:35,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:37,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:37,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:54:37,247 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:54:37,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:54:37,249 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:54:37,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:37,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:54:39,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:40,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:54:40,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:54:40,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:54:41,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:42,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:43,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:45,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:47,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:47,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:49,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:50,498 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:54:51,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:53,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:53,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:54:55,653 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:54:55,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:54:55,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:57,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:54:58,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:54:59,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:01,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:03,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:03,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:05,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:07,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:09,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:09,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:10,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:55:10,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:55:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:55:11,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:13,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:14,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:15,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:17,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:19,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:19,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:20,500 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:55:21,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:21,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:55:21,740 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:55:21,740 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:55:21,742 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:55:22,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:55:23,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:25,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:55:25,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:55:25,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:55:25,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:27,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:29,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:30,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:31,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:33,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:35,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:35,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:37,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:39,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:40,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:55:40,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:55:40,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:55:41,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:41,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:43,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:45,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:46,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:47,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:49,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:50,502 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:55:51,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:51,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:53,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:55,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:55:55,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:55:55,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:55:55,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:57,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:55:57,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:55:59,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:01,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:02,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:03,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:05,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:06,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:56:06,193 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:56:06,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:56:06,196 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:56:06,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:56:07,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:07,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:09,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:10,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:56:10,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:56:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:56:11,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:13,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:13,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:15,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:17,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:18,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:19,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:20,505 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:56:21,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:23,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:24,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:25,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:56:25,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:56:25,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:56:25,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:27,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:29,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:29,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:31,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:33,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:34,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:35,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:37,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:39,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:40,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:40,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:56:40,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:56:40,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:56:41,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:43,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:45,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:45,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:47,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:49,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:50,507 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:56:50,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:50,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:56:50,820 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:56:50,821 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:56:50,822 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:56:50,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:56:51,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:53,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:55,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:56:55,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:56:55,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:56:55,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:56:55,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:57,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:56:59,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:00,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:01,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:03,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:05,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:06,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:07,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:09,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:10,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:57:10,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:57:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:57:11,732 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:11,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:13,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:15,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:16,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:17,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:19,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:20,509 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:57:21,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:22,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:23,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:25,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:57:25,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:57:25,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:57:25,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:27,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:27,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:29,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:31,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:32,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:33,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:35,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:57:35,535 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:57:35,536 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:57:35,537 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:57:35,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:35,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:57:37,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:38,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:39,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:40,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:57:40,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:57:40,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:57:41,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:43,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:43,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:45,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:47,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:48,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:49,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:50,511 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:57:51,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:53,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:54,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:55,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:57:55,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:57:55,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:57:55,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:57,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:57:59,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:57:59,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:01,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:03,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:05,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:05,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:07,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:09,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:10,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:10,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:58:10,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:58:10,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:58:11,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:13,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:15,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:15,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:17,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:19,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:20,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:58:20,230 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:58:20,231 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:58:20,233 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:58:20,512 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:58:20,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:58:21,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:21,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:23,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:25,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:58:25,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:58:25,667 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:58:25,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:26,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:27,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:29,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:31,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:32,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:34,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:36,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:37,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:38,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:40,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:40,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:58:40,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:58:40,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:58:42,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:42,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:44,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:46,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:48,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:48,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:50,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:50,515 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:58:52,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:53,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:58:54,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:55,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:58:55,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:58:55,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:58:56,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:58,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:58:59,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:00,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:02,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:04,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:04,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:05,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:59:05,360 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:59:05,361 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:59:05,362 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:59:06,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:06,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:59:08,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:09,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:10,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:10,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:59:10,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:59:10,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:59:12,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:14,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:15,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:16,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:18,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:20,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:20,517 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:59:20,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:22,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:24,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:25,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:59:25,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:59:25,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:59:25,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:26,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:28,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:30,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:30,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:32,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:34,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:36,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:36,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:38,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:40,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:40,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:59:40,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:59:40,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:59:41,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:42,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:44,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:46,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:46,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:48,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:50,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:50,531 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 19:59:51,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 19:59:51,381 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 19:59:51,381 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 19:59:51,383 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 19:59:52,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:52,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 19:59:52,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:54,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:55,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 19:59:55,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 19:59:55,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 19:59:56,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 19:59:57,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 19:59:58,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:00,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:02,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:03,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:04,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:06,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:08,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:09,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:10,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:10,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:00:10,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:00:10,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:00:12,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:14,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:14,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:16,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:18,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:19,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:20,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:20,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:00:22,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:24,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:25,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:25,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:00:25,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:00:25,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:00:28,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:30,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:30,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:32,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:34,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:36,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:36,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:38,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:40,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:40,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:00:40,660 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:00:40,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:00:41,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:00:41,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:00:41,111 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:00:41,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:00:41,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:41,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:00:42,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:44,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:46,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:47,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:48,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:50,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:50,523 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:00:52,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:53,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:00:54,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:55,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:00:55,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:00:55,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:00:56,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:58,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:00:59,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:00,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:02,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:04,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:05,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:06,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:08,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:10,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:10,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:01:10,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:01:10,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:01:10,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:12,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:14,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:15,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:16,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:18,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:20,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:20,526 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:01:21,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:22,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:24,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:25,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:01:25,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:01:25,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:01:26,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:26,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:28,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:29,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:01:29,223 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:01:29,223 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:01:29,225 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:01:29,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:29,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:01:32,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:32,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:34,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:36,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:37,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:38,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:40,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:40,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:01:40,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:01:40,805 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:01:42,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:43,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:44,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:46,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:48,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:49,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:50,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:50,539 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:01:52,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:54,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:54,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:01:55,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:01:55,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:01:55,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:01:56,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:01:58,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:00,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:00,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:02,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:04,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:06,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:06,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:08,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:10,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:10,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:02:10,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:02:10,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:02:12,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:12,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:14,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:16,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:16,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:02:16,847 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:02:16,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:02:16,849 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:02:17,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:17,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:02:17,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:20,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:20,530 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:02:22,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:23,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:24,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:25,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:02:25,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:02:25,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:02:26,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:28,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:29,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:30,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:32,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:34,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:34,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:36,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:38,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:40,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:40,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:40,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:02:40,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:02:40,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:02:42,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:44,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:46,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:46,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:48,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:50,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:50,532 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:02:51,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:52,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:54,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:55,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:02:55,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:02:55,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:02:56,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:02:57,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:02:58,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:00,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:02,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:03,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:04,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:03:04,019 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:03:04,020 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:03:04,023 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:03:04,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:04,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:03:05,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:08,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:08,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:10,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:10,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:03:10,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:03:10,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:03:12,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:14,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:14,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:16,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:18,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:20,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:20,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:20,535 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:03:22,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:24,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:25,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:25,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:03:25,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:03:25,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:03:26,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:28,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:30,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:31,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:32,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:34,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:36,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:36,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:38,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:40,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:40,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:03:40,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:03:40,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:03:42,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:42,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:44,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:46,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:48,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:48,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:50,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:50,549 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:03:50,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:03:50,953 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:03:50,953 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:03:50,955 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:03:51,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:51,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:03:53,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:03:54,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:55,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:03:55,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:03:55,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:03:56,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:58,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:03:59,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:00,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:02,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:04,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:05,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:06,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:08,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:10,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:10,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:10,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:04:10,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:04:10,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:04:12,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:14,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:16,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:16,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:18,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:20,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:20,540 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:04:21,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:22,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:24,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:25,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:04:25,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:04:25,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:04:26,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:27,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:28,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:30,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:32,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:33,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:34,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:36,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:37,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:04:37,622 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:04:37,622 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:04:37,624 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:04:38,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:38,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:04:38,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:39,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:40,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:04:40,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:04:40,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:04:42,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:44,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:44,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:46,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:48,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:49,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:50,542 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:04:50,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:52,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:54,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:55,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:04:55,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:04:55,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:04:55,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:04:56,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:04:58,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:00,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:00,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:02,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:04,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:06,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:06,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:08,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:10,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:10,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:05:10,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:05:10,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:05:12,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:12,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:14,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:16,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:17,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:18,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:20,544 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:05:20,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:22,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:23,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:24,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:05:24,173 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:05:24,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:05:24,175 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:05:24,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:24,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:05:25,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:25,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:05:25,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:05:25,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:05:28,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:28,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:30,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:32,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:34,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:34,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:36,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:38,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:39,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:40,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:40,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:05:40,787 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:05:40,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:05:42,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:44,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:45,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:46,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:48,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:50,556 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:05:50,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:50,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:52,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:54,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:55,692 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:05:55,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:05:55,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:05:55,819 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:05:56,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:05:58,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:00,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:01,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:02,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:04,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:06,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:06,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:08,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:10,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:06:10,548 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:06:10,549 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:06:10,550 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:06:10,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:10,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:06:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:06:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:06:10,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:06:11,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:11,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:14,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:16,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:17,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:18,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:19,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:20,548 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:06:21,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:22,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:23,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:25,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:25,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:06:25,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:06:25,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:06:27,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:28,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:29,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:31,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:33,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:33,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:35,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:37,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:39,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:39,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:40,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:06:40,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:06:40,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:06:41,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:43,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:44,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:45,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:47,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:49,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:50,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:50,551 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:06:51,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:53,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:55,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:55,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:06:55,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:06:55,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:06:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:06:56,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:06:56,708 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:06:56,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:06:56,710 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:06:57,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:06:57,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:06:59,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:01,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:01,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:03,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:05,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:06,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:07,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:09,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:07:10,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:07:10,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:07:11,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:12,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:13,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:15,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:17,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:17,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:19,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:20,553 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:07:21,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:23,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:23,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:25,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:25,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:07:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:07:25,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:07:27,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:28,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:29,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:31,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:33,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:34,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:35,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:37,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:39,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:39,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:40,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:07:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:07:40,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:07:41,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:42,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:07:42,667 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:07:42,667 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:07:42,669 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:07:42,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:07:43,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:45,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:45,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:47,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:49,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:50,567 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:07:50,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:51,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:53,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:55,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:07:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:07:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:07:55,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:07:57,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:07:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:00,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:03,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:05,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:06,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:07,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:10,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:08:10,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:08:10,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:08:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:11,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:13,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:17,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:19,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:20,558 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:08:21,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:22,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:23,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:25,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:08:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:08:25,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:08:27,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:28,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:28,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:08:28,454 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:08:28,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:08:28,455 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:08:28,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:08:29,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:31,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:33,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:33,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:35,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:37,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:39,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:39,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:08:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:08:40,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:08:41,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:43,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:44,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:45,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:47,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:49,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:50,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:50,560 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:08:51,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:53,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:08:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:08:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:08:55,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:08:55,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:57,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:08:59,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:01,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:01,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:03,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:05,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:06,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:07,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:09,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:10,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:09:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:09:10,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:09:11,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:11,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:13,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:14,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:09:14,229 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:09:14,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:09:14,231 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:09:14,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:09:16,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:17,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:18,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:20,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:20,562 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:09:22,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:22,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:24,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:09:25,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:09:25,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:09:26,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:27,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:28,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:30,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:32,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:33,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:34,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:36,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:38,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:38,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:40,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:40,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:09:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:09:40,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:09:42,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:44,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:44,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:46,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:48,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:50,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:50,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:50,575 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:09:52,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:54,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:55,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:09:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:09:55,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:09:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:09:56,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:09:58,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:00,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:00,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:10:00,092 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:10:00,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:10:00,094 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:10:01,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:10:01,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:02,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:04,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:06,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:06,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:08,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:10,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:10:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:10:10,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:10:12,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:12,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:14,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:16,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:17,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:18,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:20,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:20,567 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:10:22,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:22,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:24,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:10:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:10:25,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:10:26,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:28,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:28,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:30,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:32,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:33,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:34,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:36,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:38,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:39,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:40,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:10:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:10:40,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:10:42,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:44,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:44,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:45,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:10:45,637 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:10:45,638 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:10:45,639 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:10:46,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:46,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:10:48,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:50,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:50,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:50,568 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:10:52,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:54,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:55,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:10:55,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:10:55,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:10:55,800 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:10:56,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:10:58,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:00,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:01,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:02,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:04,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:06,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:06,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:08,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:10,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:11:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:11:10,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:11:12,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:12,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:14,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:16,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:17,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:18,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:20,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:20,571 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:11:22,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:23,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:24,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:11:25,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:11:25,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:11:26,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:28,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:28,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:30,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:31,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:11:31,323 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:11:31,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:11:31,325 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:11:32,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:32,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:11:34,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:34,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:36,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:38,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:39,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:40,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:11:40,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:11:40,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:11:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:44,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:45,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:46,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:48,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:50,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:50,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:50,585 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:11:52,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:54,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:55,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:11:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:11:55,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:11:55,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:11:56,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:11:58,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:00,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:01,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:02,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:04,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:06,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:06,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:08,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:10,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:10,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:12:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:12:10,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:12:12,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:12,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:14,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:16,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:16,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:12:16,875 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:12:16,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:12:16,877 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:12:17,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:12:17,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:18,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:20,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:20,620 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:12:22,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:24,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:12:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:12:25,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:12:26,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:28,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:28,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:30,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:32,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:34,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:34,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:36,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:38,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:39,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:40,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:12:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:12:40,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:12:42,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:44,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:45,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:46,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:48,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:50,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:50,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:50,622 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:12:52,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:54,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:12:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:12:55,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:12:55,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:12:56,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:12:58,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:00,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:01,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:02,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:02,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:13:02,319 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:13:02,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:13:02,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:13:03,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:13:04,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:06,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:06,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:08,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:10,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:13:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:13:10,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:13:12,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:12,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:14,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:16,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:17,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:18,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:20,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:20,624 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:13:22,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:23,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:24,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:13:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:13:25,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:13:26,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:28,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:28,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:30,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:32,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:34,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:34,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:36,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:38,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:39,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:40,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:13:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:13:40,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:13:42,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:44,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:45,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:46,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:47,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:13:47,738 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:13:47,739 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:13:47,741 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:13:48,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:48,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:13:50,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:50,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:50,638 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:13:52,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:54,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:55,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:13:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:13:55,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:13:55,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:13:56,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:13:58,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:00,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:01,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:02,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:04,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:06,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:06,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:08,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:10,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:10,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:14:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:14:10,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:14:12,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:12,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:14,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:16,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:17,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:18,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:20,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:20,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:14:22,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:23,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:24,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:25,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:14:25,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:14:25,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:14:26,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:28,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:28,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:30,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:32,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:33,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:14:33,110 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:14:33,110 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:14:33,112 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:14:33,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:14:34,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:34,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:36,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:38,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:39,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:40,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:40,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:14:40,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:14:40,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:14:42,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:44,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:45,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:46,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:48,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:50,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:50,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:50,632 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:14:52,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:54,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:14:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:14:55,802 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:14:55,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:14:56,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:14:58,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:00,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:01,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:02,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:04,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:06,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:06,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:08,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:10,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:15:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:15:10,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:15:12,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:12,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:14,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:16,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:17,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:18,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:18,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:15:18,572 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:15:18,573 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:15:18,574 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:15:19,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:19,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:15:20,634 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:15:22,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:23,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:24,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:25,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:15:25,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:15:25,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:15:26,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:28,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:28,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:30,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:32,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:34,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:34,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:36,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:38,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:39,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:40,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:40,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:15:40,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:15:40,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:15:42,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:44,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:46,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:48,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:50,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:50,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:50,649 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:15:52,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:54,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:15:55,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:15:55,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:15:55,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:15:56,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:15:58,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:00,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:01,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:02,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:04,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:16:04,011 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:16:04,012 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:16:04,014 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:16:04,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:04,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:16:05,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:06,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:08,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:10,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:16:10,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:16:10,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:16:12,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:12,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:14,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:16,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:17,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:18,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:20,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:20,638 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:16:22,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:23,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:24,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:25,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:16:25,854 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:16:25,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:16:26,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:28,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:28,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:30,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:32,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:34,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:34,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:36,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:38,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:39,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:40,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:40,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:16:40,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:16:40,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:16:42,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:44,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:44,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:46,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:48,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:49,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:16:49,368 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:16:49,368 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:16:49,370 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:16:49,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:49,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:16:50,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:50,641 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:16:52,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:54,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:55,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:16:55,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:16:55,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:16:55,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:16:56,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:16:58,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:00,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:01,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:02,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:04,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:06,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:06,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:08,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:10,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:10,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:17:10,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:17:10,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:17:12,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:12,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:14,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:16,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:17,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:18,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:20,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:20,643 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:17:22,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:23,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:24,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:25,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:17:25,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:17:25,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:17:26,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:28,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:28,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:30,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:32,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:33,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:34,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:34,732 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:17:34,734 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:17:34,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:17:34,736 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:17:35,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:35,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:17:38,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:39,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:40,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:17:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:17:40,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:17:42,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:44,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:44,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:46,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:48,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:50,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:50,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:50,657 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:17:52,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:54,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:55,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:17:55,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:17:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:17:55,848 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:17:56,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:17:58,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:00,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:01,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:02,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:04,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:06,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:06,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:08,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:10,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:10,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:18:10,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:18:10,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:18:11,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:12,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:14,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:16,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:17,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:18,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:20,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:18:20,085 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:18:20,085 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:18:20,087 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:18:20,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:20,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:18:20,647 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:18:21,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:22,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:24,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:25,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:18:25,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:18:25,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:18:26,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:28,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:28,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:30,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:32,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:33,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:34,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:36,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:38,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:39,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:40,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:18:40,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:18:40,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:18:42,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:44,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:44,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:46,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:48,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:49,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:50,649 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:18:50,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:52,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:54,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:55,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:18:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:18:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:18:55,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:18:56,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:18:58,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:00,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:00,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:02,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:04,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:05,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:19:05,373 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:19:05,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:19:05,375 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:19:05,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:05,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:19:06,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:08,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:10,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:10,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:19:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:19:10,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:19:11,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:12,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:14,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:16,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:17,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:18,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:20,651 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:19:20,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:22,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:22,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:24,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:25,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:19:25,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:19:25,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:19:26,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:28,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:28,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:30,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:32,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:33,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:34,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:36,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:38,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:38,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:40,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:19:40,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:19:40,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:19:40,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:41,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:43,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:44,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:45,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:47,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:49,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:49,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:50,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:19:50,728 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:19:50,728 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:19:50,728 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:19:50,730 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:19:50,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:19:51,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:53,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:55,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:19:55,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:19:55,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:19:55,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:19:55,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:57,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:19:59,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:00,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:01,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:03,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:05,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:06,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:07,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:09,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:10,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:20:10,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:20:10,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:20:11,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:11,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:13,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:15,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:17,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:17,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:19,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:20,656 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:20:21,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:22,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:23,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:25,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:20:25,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:20:25,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:20:25,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:27,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:28,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:29,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:31,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:33,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:33,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:35,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:36,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:20:36,177 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:20:36,177 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:20:36,179 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:20:36,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:20:37,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:38,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:39,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:40,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:20:40,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:20:40,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:20:41,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:43,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:44,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:47,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:49,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:49,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:50,658 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:20:51,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:53,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:20:55,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:20:55,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:20:55,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:20:55,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:20:59,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:00,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:01,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:03,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:05,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:06,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:07,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:09,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:10,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:21:10,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:21:10,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:21:11,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:11,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:13,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:15,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:17,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:18,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:20,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:20,660 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:21:21,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:21:21,657 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:21:21,658 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:21:21,659 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:21:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:21:22,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:25,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:21:25,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:21:25,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:21:26,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:28,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:28,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:30,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:32,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:33,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:34,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:36,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:38,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:39,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:40,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:40,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:21:40,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:21:40,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:21:42,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:44,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:44,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:46,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:48,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:50,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:50,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:50,711 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:21:52,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:54,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:55,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:21:55,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:21:55,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:21:55,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:21:56,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:21:58,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:00,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:00,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:02,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:04,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:06,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:06,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:07,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:22:07,273 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:22:07,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:22:07,275 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:22:08,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:08,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:22:10,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:10,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:22:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:22:10,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:22:12,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:12,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:14,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:16,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:17,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:18,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:20,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:20,665 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:22:22,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:22,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:24,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:25,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:22:25,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:22:25,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:22:26,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:28,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:28,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:30,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:32,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:33,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:34,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:36,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:38,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:39,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:40,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:40,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:22:40,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:22:40,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:22:42,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:44,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:44,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:46,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:48,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:50,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:50,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:50,666 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:22:52,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:52,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:22:52,793 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:22:52,793 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:22:52,795 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:22:53,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:22:54,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:55,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:22:55,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:22:55,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:22:55,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:22:56,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:22:58,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:00,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:01,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:02,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:04,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:06,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:06,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:08,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:10,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:10,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:23:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:23:10,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:23:12,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:12,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:14,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:16,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:17,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:18,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:20,669 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:23:22,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:22,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:24,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:25,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:23:25,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:23:25,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:23:26,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:28,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:28,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:30,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:32,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:33,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:34,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:36,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:38,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:38,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:23:38,381 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:23:38,381 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:23:38,383 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:23:39,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:23:39,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:40,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:40,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:23:40,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:23:40,869 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:23:42,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:44,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:44,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:46,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:48,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:50,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:50,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:50,670 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:23:52,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:54,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:55,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:23:55,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:23:55,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:23:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:23:56,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:23:58,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:00,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:01,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:02,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:04,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:06,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:06,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:08,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:10,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:10,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:24:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:24:10,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:24:12,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:12,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:14,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:16,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:17,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:18,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:20,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:20,673 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:24:22,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:22,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:23,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:24:23,814 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:24:23,815 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:24:23,818 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:24:24,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:24,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:24:25,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:24:25,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:24:25,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:24:26,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:28,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:28,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:30,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:32,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:33,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:34,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:36,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:38,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:39,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:40,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:40,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:24:40,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:24:40,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:24:42,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:44,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:44,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:46,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:48,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:50,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:50,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:50,675 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:24:52,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:54,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:55,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:24:55,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:24:55,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:24:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:24:56,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:24:58,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:00,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:01,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:02,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:04,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:06,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:06,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:08,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:09,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:25:09,275 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:25:09,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:25:09,277 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:25:10,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:10,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:25:10,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:25:10,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:25:10,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:25:12,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:12,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:14,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:16,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:17,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:18,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:20,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:20,677 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:25:22,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:22,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:24,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:25,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:25:25,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:25:25,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:25:26,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:28,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:28,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:30,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:32,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:33,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:34,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:36,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:38,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:39,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:40,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:40,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:25:40,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:25:40,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:25:42,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:44,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:44,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:46,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:48,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:50,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:50,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:50,680 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:25:52,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:54,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:54,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:25:54,688 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:25:54,688 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:25:54,690 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:25:55,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:25:55,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:25:55,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:25:55,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:25:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:25:56,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:25:58,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:00,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:01,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:02,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:04,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:06,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:06,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:08,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:10,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:10,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:26:10,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:26:10,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:26:12,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:12,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:14,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:16,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:17,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:18,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:20,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:20,682 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:26:22,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:23,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:24,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:25,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:26:25,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:26:25,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:26:26,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:28,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:28,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:30,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:32,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:34,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:34,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:36,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:38,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:39,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:40,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:40,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:26:40,404 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:26:40,405 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:26:40,406 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:26:40,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:26:40,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:26:40,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:26:41,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:26:42,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:44,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:45,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:46,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:48,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:50,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:50,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:50,684 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:26:52,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:54,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:55,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:26:55,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:26:55,870 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:26:55,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:26:56,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:26:58,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:00,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:01,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:02,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:04,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:06,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:06,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:08,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:10,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:10,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:27:10,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:27:10,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:27:12,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:12,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:14,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:16,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:17,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:18,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:20,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:20,686 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:27:22,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:23,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:24,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:25,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:27:25,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:27:25,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:27:26,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:27:26,137 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:27:26,137 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:27:26,139 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:27:26,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:26,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:27:28,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:28,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:30,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:32,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:34,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:34,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:36,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:38,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:39,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:40,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:40,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:27:40,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:27:40,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:27:42,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:44,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:45,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:46,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:48,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:50,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:50,690 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:27:50,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:52,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:54,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:55,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:27:55,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:27:55,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:27:56,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:27:56,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:27:58,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:00,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:01,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:02,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:04,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:06,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:07,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:08,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:10,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:10,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:28:10,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:28:10,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:28:11,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:28:11,779 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:28:11,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:28:11,781 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:28:12,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:12,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:28:12,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:14,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:16,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:18,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:18,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:20,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:20,692 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:28:22,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:23,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:24,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:25,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:28:25,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:28:25,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:28:26,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:28,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:29,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:30,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:32,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:34,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:34,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:36,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:38,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:40,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:40,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:40,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:28:40,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:28:40,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:28:42,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:44,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:45,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:46,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:48,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:50,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:50,694 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:28:50,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:52,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:54,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:55,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:28:55,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:28:55,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:28:55,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:28:56,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:57,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:28:57,315 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:28:57,315 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:28:57,317 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:28:57,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:28:58,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:28:59,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:00,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:01,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:02,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:04,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:06,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:07,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:08,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:10,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:10,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:29:10,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:29:10,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:29:12,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:12,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:14,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:16,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:18,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:18,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:20,697 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:29:20,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:22,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:23,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:24,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:25,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:29:25,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:29:25,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:29:26,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:28,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:29,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:30,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:32,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:34,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:34,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:36,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:38,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:40,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:40,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:40,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:29:40,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:29:40,871 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:29:42,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:42,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:29:42,783 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:29:42,783 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:29:42,785 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:29:43,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:43,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:29:44,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:45,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:46,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:48,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:50,699 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:29:50,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:50,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:52,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:54,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:55,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:29:55,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:29:55,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:29:56,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:29:56,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:29:58,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:00,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:01,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:02,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:04,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:06,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:07,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:08,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:10,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:30:10,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:30:10,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:30:12,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:12,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:14,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:16,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:18,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:18,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:20,701 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:30:20,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:22,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:23,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:24,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:25,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:30:25,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:30:25,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:30:26,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:28,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:30:28,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:30:28,111 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:30:28,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:30:28,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:28,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:30:29,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:29,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:30,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:32,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:34,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:34,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:36,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:38,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:39,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:40,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:40,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:30:40,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:30:40,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:30:42,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:44,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:45,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:46,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:48,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:50,703 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:30:50,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:50,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:52,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:54,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:55,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:30:55,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:30:55,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:30:55,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:30:56,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:30:58,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:00,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:01,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:02,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:04,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:06,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:07,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:08,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:10,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:31:10,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:31:10,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:31:12,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:12,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:13,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:31:13,471 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:31:13,472 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:31:13,473 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:31:13,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:31:14,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:15,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:16,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:18,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:18,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:20,705 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:31:20,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:22,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:23,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:24,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:25,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:31:25,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:31:25,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:31:26,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:28,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:28,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:30,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:32,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:34,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:34,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:36,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:38,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:39,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:40,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:31:40,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:31:40,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:31:40,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:42,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:44,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:45,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:46,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:48,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:50,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:50,707 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:31:50,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:52,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:54,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:55,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:31:55,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:31:55,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:31:55,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:31:56,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:58,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:31:58,850 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:31:58,850 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:31:58,852 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:31:58,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:31:58,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:31:59,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:00,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:01,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:02,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:04,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:06,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:07,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:08,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:32:10,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:32:10,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:32:10,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:12,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:12,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:14,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:16,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:17,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:18,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:20,709 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:32:20,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:22,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:23,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:24,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:25,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:32:25,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:32:25,872 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:32:26,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:28,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:28,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:30,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:32,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:34,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:34,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:36,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:38,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:39,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:40,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:32:40,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:32:40,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:32:40,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:42,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:44,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:32:44,129 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:32:44,130 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:32:44,132 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:32:44,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:44,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:32:45,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:45,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:46,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:48,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:50,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:50,712 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:32:50,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:52,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:54,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:55,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:32:55,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:32:55,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:32:56,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:32:56,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:32:58,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:00,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:01,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:02,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:04,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:06,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:06,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:08,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:10,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:33:10,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:33:10,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:33:10,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:12,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:12,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:14,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:16,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:17,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:18,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:20,714 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:33:20,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:22,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:23,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:24,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:25,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:33:25,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:33:25,894 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:33:26,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:28,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:28,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:29,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:33:29,439 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:33:29,440 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:33:29,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:33:29,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:33:30,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:32,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:33,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:34,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:35,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:37,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:39,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:39,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:40,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:33:40,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:33:40,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:33:41,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:43,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:44,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:45,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:47,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:49,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:50,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:50,716 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:33:51,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:53,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:55,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:55,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:33:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:33:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:33:55,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:33:57,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:33:59,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:01,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:01,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:03,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:05,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:06,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:07,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:09,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:10,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:34:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:34:10,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:34:11,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:12,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:13,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:14,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:34:14,714 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:34:14,714 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:34:14,716 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:34:15,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:15,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:34:16,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:17,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:17,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:19,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:20,718 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:34:21,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:23,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:25,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:25,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:34:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:34:25,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:34:27,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:28,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:29,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:31,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:33,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:33,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:35,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:37,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:39,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:39,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:40,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:34:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:34:40,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:34:41,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:43,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:44,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:45,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:47,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:49,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:50,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:50,721 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:34:51,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:53,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:55,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:55,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:34:55,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:34:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:34:55,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:34:57,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:59,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:34:59,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:34:59,919 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:34:59,919 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:34:59,922 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:35:00,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:35:00,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:01,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:02,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:03,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:05,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:06,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:35:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:35:10,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:35:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:12,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:13,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:15,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:17,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:17,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:19,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:20,722 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:35:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:22,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:23,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:25,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:35:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:35:25,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:35:27,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:28,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:29,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:31,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:33,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:33,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:35,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:37,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:38,896 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:39,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:35:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:35:40,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:35:41,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:43,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:44,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:45,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:45,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:35:45,159 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:35:45,159 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:35:45,161 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:35:46,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:35:47,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:48,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:49,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:49,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:50,725 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:35:51,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:53,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:55,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:35:55,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:35:55,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:35:55,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:35:57,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:35:59,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:00,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:01,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:03,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:05,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:05,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:07,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:09,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:36:10,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:36:10,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:36:11,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:11,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:13,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:15,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:16,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:17,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:19,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:20,742 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:36:21,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:22,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:23,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:25,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:25,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:36:25,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:36:25,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:36:27,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:27,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:29,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:30,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:36:30,299 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:36:30,299 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:36:30,302 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:36:31,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:31,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:36:32,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:33,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:34,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:36,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:38,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:38,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:40,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:40,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:36:40,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:36:40,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:36:42,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:43,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:44,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:46,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:48,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:49,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:50,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:50,730 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:36:52,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:54,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:54,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:36:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:36:55,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:36:55,876 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:36:56,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:36:58,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:00,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:00,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:02,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:04,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:05,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:06,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:08,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:10,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:10,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:37:10,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:37:10,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:37:10,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:12,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:14,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:15,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:37:15,500 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:37:15,501 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:37:15,502 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:37:16,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:16,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:37:16,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:18,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:20,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:20,732 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:37:21,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:22,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:24,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:25,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:37:25,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:37:25,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:37:26,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:27,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:28,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:30,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:32,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:32,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:34,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:36,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:38,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:38,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:40,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:40,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:37:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:37:40,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:37:42,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:43,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:44,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:46,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:48,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:48,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:50,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:50,734 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:37:52,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:54,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:54,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:37:55,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:37:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:37:55,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:37:56,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:58,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:37:59,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:00,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:00,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:38:00,647 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:38:00,647 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:38:00,649 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:38:01,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:38:02,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:04,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:05,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:06,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:08,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:10,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:10,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:38:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:38:10,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:38:12,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:14,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:16,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:16,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:18,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:20,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:20,737 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:38:21,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:22,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:24,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:38:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:38:25,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:38:26,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:27,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:28,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:30,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:32,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:32,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:34,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:36,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:37,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:38,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:40,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:38:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:38:40,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:38:42,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:43,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:44,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:45,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:38:45,908 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:38:45,909 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:38:45,911 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:38:46,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:46,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:38:48,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:48,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:50,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:50,739 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:38:52,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:54,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:38:54,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:38:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:38:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:38:56,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:58,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:38:59,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:00,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:02,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:04,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:04,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:06,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:08,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:10,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:10,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:39:10,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:39:10,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:39:12,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:14,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:15,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:16,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:18,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:20,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:20,741 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:39:21,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:22,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:24,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:39:25,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:39:25,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:39:26,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:27,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:28,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:30,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:31,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:39:31,117 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:39:31,117 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:39:31,119 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:39:31,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:39:32,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:32,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:34,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:36,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:37,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:38,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:40,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:39:40,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:39:40,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:39:42,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:42,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:44,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:46,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:48,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:48,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:50,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:50,743 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:39:52,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:53,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:39:54,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:39:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:39:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:39:56,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:58,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:39:59,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:00,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:02,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:04,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:04,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:06,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:08,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:09,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:10,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:40:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:40:10,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:40:12,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:14,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:15,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:16,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:40:16,229 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:40:16,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:40:16,231 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:40:16,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:16,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:40:17,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:18,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:20,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:20,745 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:40:20,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:22,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:24,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:40:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:40:25,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:40:25,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:26,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:28,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:30,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:31,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:32,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:34,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:36,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:37,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:38,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:40,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:40:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:40:40,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:40:42,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:42,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:44,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:46,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:47,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:48,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:50,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:50,748 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:40:52,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:53,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:54,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:55,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:40:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:40:55,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:40:56,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:40:58,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:40:58,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:00,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:01,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:41:01,337 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:41:01,338 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:41:01,339 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:41:01,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:41:02,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:03,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:04,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:04,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:06,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:08,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:09,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:10,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:41:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:41:10,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:41:12,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:14,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:14,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:16,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:18,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:20,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:20,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:20,750 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:41:22,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:24,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:25,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:41:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:41:25,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:41:26,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:28,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:30,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:31,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:32,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:34,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:36,569 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:36,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:38,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:40,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:41:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:41:40,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:41:42,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:42,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:44,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:46,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:41:46,419 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:41:46,419 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:41:46,420 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:41:46,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:46,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:41:47,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:47,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:48,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:50,752 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:41:50,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:52,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:52,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:54,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:55,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:41:55,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:41:55,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:41:56,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:41:58,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:41:58,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:00,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:02,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:03,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:04,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:06,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:08,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:09,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:10,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:10,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:42:10,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:42:10,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:42:12,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:14,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:14,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:16,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:18,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:20,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:20,767 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:42:20,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:22,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:24,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:25,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:25,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:42:25,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:42:25,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:42:26,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:28,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:30,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:30,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:31,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:42:31,668 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:42:31,668 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:42:31,670 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:42:31,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:42:32,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:33,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:34,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:36,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:36,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:38,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:40,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:40,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:42:40,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:42:40,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:42:42,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:42,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:44,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:46,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:47,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:48,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:50,757 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:42:50,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:52,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:53,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:54,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:42:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:42:55,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:42:56,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:42:58,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:42:58,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:00,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:02,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:04,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:04,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:06,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:08,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:09,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:10,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:10,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:43:10,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:43:10,883 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:43:12,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:14,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:14,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:16,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:43:16,715 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:43:16,715 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:43:16,717 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:43:16,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:16,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:43:17,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:18,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:20,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:20,759 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:43:20,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:22,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:24,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:25,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:25,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:43:25,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:43:25,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:43:26,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:28,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:30,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:31,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:32,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:34,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:36,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:36,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:38,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:40,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:40,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:43:40,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:43:40,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:43:41,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:42,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:44,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:46,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:47,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:48,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:50,761 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:43:50,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:52,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:43:52,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:54,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:43:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 20:43:55,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:43:55,883 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 20:43:56,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:01,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:44:01,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:06,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:10,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:44:16,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:17,146 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:44:17,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:17,146 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:44:17,149 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:44:17,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:17,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:44:18,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:20,765 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:44:20,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:21,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:22,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:22,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:24,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:25,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:44:26,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:28,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:28,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:30,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:30,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:32,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:33,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:34,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:35,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:36,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:38,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:38,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:40,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:44:40,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:42,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:44,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:44,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:45,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:46,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:47,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:44:47,055 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:44:47,055 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:44:47,056 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:44:47,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:47,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:44:48,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:49,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:50,767 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:44:50,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:44:50,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:52,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:54,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:55,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:44:55,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:44:56,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:44:58,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:00,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:00,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:00,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:02,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:04,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:05,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:06,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:06,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:08,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:10,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:45:10,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:11,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:12,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:14,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:15,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:16,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:16,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:18,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:20,769 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:45:20,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:20,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:22,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:22,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:24,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:25,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:45:26,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:27,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:28,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:30,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:31,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:32,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:45:32,129 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:45:32,130 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:45:32,133 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:45:33,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:33,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:45:33,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:34,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:35,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:35,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:37,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:38,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:39,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:40,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:45:41,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:43,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:43,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:45,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:45,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:47,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:49,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:49,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:50,771 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:45:50,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:45:51,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:53,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:54,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:45:55,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:55,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:45:57,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:45:59,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:00,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:00,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:01,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:03,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:05,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:05,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:05,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:07,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:09,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:10,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:46:10,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:11,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:13,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:15,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:15,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:16,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:17,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:17,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:46:17,218 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:46:17,218 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:46:17,220 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:46:18,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:46:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:20,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:20,773 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:46:20,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:21,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:21,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:23,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:25,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:25,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:46:27,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:27,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:29,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:31,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:31,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:32,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:33,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:35,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:36,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:37,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:38,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:39,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:40,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:46:41,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:43,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:43,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:45,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:46,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:47,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:48,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:49,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:50,776 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:46:51,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:46:51,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:53,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:54,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:46:55,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:55,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:46:57,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:59,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:46:59,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:01,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:01,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:02,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:47:02,261 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:47:02,262 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:47:02,263 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:47:03,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:03,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:47:04,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:05,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:05,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:06,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:09,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:10,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:10,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:47:11,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:13,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:15,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:15,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:16,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:17,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:19,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:20,777 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:47:21,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:21,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:21,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:23,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:25,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:25,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:47:27,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:27,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:29,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:31,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:31,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:32,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:33,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:35,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:36,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:37,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:38,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:39,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:40,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:47:41,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:43,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:43,687 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:45,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:46,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:47,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:47,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:47:47,315 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:47:47,316 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:47:47,318 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:47:48,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:47:49,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:49,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:50,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:50,780 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:47:51,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:47:51,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:53,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:54,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:47:55,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:55,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:47:57,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:59,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:47:59,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:01,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:01,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:03,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:05,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:05,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:06,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:07,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:09,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:10,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:10,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:48:11,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:13,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:15,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:16,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:16,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:17,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:19,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:20,793 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:48:21,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:21,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:21,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:23,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:25,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:25,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:48:26,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:27,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:29,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:31,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:31,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:32,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:32,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:48:32,405 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:48:32,406 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:48:32,407 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:48:33,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:33,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:48:34,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:35,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:36,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:37,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:37,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:39,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:40,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:48:41,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:43,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:43,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:45,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:46,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:47,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:48,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:49,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:50,785 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:48:51,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:48:51,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:53,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:54,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:48:55,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:55,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:48:57,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:59,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:48:59,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:01,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:01,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:03,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:04,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:05,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:06,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:07,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:09,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:10,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:10,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:49:11,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:13,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:15,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:15,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:16,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:17,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:17,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:49:17,579 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:49:17,579 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:49:17,581 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:49:18,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:18,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:49:20,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:20,788 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:49:20,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:21,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:22,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:24,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:25,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:49:26,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:26,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:28,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:30,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:31,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:31,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:32,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:34,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:36,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:36,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:37,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:38,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:40,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:40,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:49:42,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:42,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:44,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:46,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:46,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:48,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:48,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:50,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:50,789 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:49:51,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:49:52,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:53,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:49:54,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:55,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:49:56,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:58,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:49:59,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:00,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:01,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:02,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:02,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:50:02,698 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:50:02,698 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:50:02,700 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:50:03,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:50:04,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:04,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:06,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:06,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:08,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:09,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:10,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:10,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:50:12,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:14,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:15,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:16,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:16,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:18,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:20,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:20,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:20,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:50:21,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:22,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:24,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:25,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:50:26,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:26,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:28,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:30,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:31,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:31,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:32,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:34,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:36,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:36,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:36,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:38,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:40,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:40,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:50:42,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:42,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:44,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:46,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:46,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:47,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:47,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:50:47,701 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:50:47,701 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:50:47,703 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:50:48,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:48,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:50:50,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:50,793 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:50:51,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:50:52,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:53,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:54,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:55,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:50:56,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:50:58,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:50:58,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:00,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:01,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:02,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:03,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:04,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:06,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:06,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:08,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:09,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:10,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:10,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:51:12,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:14,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:14,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:16,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:16,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:18,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:20,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:20,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:20,796 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:51:21,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:22,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:24,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:25,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:25,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:51:26,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:28,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:30,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:30,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:31,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:32,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:32,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:51:32,765 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:51:32,766 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:51:32,767 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:51:33,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:51:34,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:36,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:36,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:36,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:38,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:40,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:40,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:51:41,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:42,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:44,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:46,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:46,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:47,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:48,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:50,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:50,798 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:51:51,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:51:52,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:52,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:54,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:55,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:51:56,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:51:58,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:51:58,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:00,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:01,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:02,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:03,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:04,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:06,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:06,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:08,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:08,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:10,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:10,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:52:12,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:14,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:14,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:16,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:16,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:17,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:52:17,900 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:52:17,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:52:17,902 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:52:18,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:18,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:52:19,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:20,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:20,811 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:52:21,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:22,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:24,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:25,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:25,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:52:26,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:28,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:30,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:30,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:31,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:32,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:34,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:35,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:36,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:36,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:38,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:40,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:40,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:52:41,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:42,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:44,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:46,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:46,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:46,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:48,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:50,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:50,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:52:51,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:52:51,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:52,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:54,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:55,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:52:56,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:52:57,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:52:58,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:00,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:01,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:02,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:02,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:02,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:53:02,970 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:53:02,971 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:53:02,972 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:53:03,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:03,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:53:04,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:06,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:06,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:08,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:08,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:10,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:10,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:53:12,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:13,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:14,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:16,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:16,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:18,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:19,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:20,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:20,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:53:21,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:22,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:24,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:24,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:25,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:53:26,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:28,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:29,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:30,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:31,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:32,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:34,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:35,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:36,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:36,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:38,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:40,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:40,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:40,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:53:42,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:44,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:46,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:46,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:46,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:48,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:53:48,096 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:53:48,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:53:48,098 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:53:48,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:48,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:53:49,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:50,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:50,807 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:53:51,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:53:51,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:52,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:54,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:55,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:53:56,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:53:57,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:53:58,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:00,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:01,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:02,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:02,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:04,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:06,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:06,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:07,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:08,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:10,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:10,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:54:12,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:13,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:14,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:16,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:16,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:18,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:18,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:20,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:20,825 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:54:21,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:22,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:24,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:24,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:25,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:54:26,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:28,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:29,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:30,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:31,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:32,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:33,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:54:33,127 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:54:33,128 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:54:33,129 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:54:33,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:33,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:54:34,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:34,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:36,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:36,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:38,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:40,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:40,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:54:42,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:44,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:45,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:46,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:46,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:48,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:50,811 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:54:50,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:50,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:51,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:54:52,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:54,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:55,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:54:56,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:54:56,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:54:58,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:00,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:01,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:01,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:02,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:04,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:06,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:06,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:07,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:08,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:10,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:10,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:55:12,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:12,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:14,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:16,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:16,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:18,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:18,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:55:18,072 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:55:18,073 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:55:18,074 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:55:18,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:18,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:55:19,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:20,813 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:55:20,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:21,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:22,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:23,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:24,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:25,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:55:26,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:28,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:28,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:30,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:31,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:32,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:34,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:34,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:36,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:36,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:38,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:39,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:40,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:40,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:55:42,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:44,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:45,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:46,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:46,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:48,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:50,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:50,816 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:55:50,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:51,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:55:52,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:54,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:55,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:55:55,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:55:56,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:55:58,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:00,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:01,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:01,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:02,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:03,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:56:03,170 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:56:03,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:56:03,172 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:56:03,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:03,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:56:04,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:06,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:06,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:06,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:08,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:10,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:10,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:56:12,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:12,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:14,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:16,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:16,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:17,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:18,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:20,831 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:56:20,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:21,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:22,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:22,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:24,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:25,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:56:26,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:28,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:28,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:30,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:31,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:32,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:33,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:34,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:36,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:36,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:38,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:39,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:40,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:41,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:56:42,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:44,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:44,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:46,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:46,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:48,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:56:48,201 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:56:48,201 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:56:48,203 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:56:48,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:48,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:56:49,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:49,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:50,821 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:56:50,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:51,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:56:52,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:54,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:56:56,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:56:56,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:56:58,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:00,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:00,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:01,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:02,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:04,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:06,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:06,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:06,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:08,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:10,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:11,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:57:11,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:12,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:14,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:16,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:16,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:16,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:18,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:20,823 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:57:20,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:21,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:22,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:22,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:24,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:26,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:57:26,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:27,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:29,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:31,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:31,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:33,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:33,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:33,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:57:33,135 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:57:33,136 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:57:33,137 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:57:34,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:34,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:57:35,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:36,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:37,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:38,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:39,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:41,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:41,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:57:43,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:43,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:45,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:46,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:47,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:49,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:49,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:50,825 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:57:51,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:51,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:57:53,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:54,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:57:55,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:56,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:57:57,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:57:59,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:00,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:01,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:01,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:03,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:05,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:05,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:06,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:07,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:09,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:10,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:11,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:58:11,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:13,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:15,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:16,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:16,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:17,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:18,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:58:18,198 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:58:18,198 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:58:18,200 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:58:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:19,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:58:20,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:20,838 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:58:21,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:21,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:21,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:23,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:25,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:26,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:58:27,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:27,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:29,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:31,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:31,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:32,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:33,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:35,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:37,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:37,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:39,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:41,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:58:41,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:43,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:43,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:45,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:46,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:47,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:48,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:49,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:50,829 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:58:51,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:51,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:58:53,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:54,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:58:55,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:56,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:58:57,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:59,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:58:59,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:01,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:01,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:03,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:03,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:59:03,263 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:59:03,263 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:59:03,265 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:59:04,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:04,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:59:05,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:05,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:06,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:07,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:09,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:10,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:11,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:59:11,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:13,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:15,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:15,835 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:16,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:17,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:19,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:20,831 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:59:21,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:21,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:21,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:23,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:25,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:26,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:59:26,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:27,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:29,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:31,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:31,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:32,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:33,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:35,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:36,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:37,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:37,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:39,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:41,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:59:41,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:42,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:43,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:45,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:46,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:47,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:48,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:48,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 20:59:48,268 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 20:59:48,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 20:59:48,270 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 20:59:48,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 20:59:49,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:50,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:50,834 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 20:59:51,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:51,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 20:59:53,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:53,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:55,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:56,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 20:59:57,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 20:59:59,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 20:59:59,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:01,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:01,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:03,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:04,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:05,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:06,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:07,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:09,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:09,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:11,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:00:11,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:13,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:15,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:15,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:16,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:17,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:19,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:20,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:20,846 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:00:21,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:21,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:23,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:25,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:26,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:00:26,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:27,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:29,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:31,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:31,450 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:31,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:33,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:00:33,287 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:00:33,288 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:00:33,289 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:00:33,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:33,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:00:34,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:35,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:36,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:36,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:37,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:39,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:41,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:00:41,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:42,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:43,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:45,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:46,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:47,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:47,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:49,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:50,838 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:00:51,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:51,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:00:53,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:53,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:55,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:56,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:00:57,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:00:58,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:00:59,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:01,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:01,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:03,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:03,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:05,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:06,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:07,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:09,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:09,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:11,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:01:11,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:13,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:14,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:15,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:16,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:17,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:18,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:01:18,352 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:01:18,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:01:18,355 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:01:18,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:01:19,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:20,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:20,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:20,840 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:01:21,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:22,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:23,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:25,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:25,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:26,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:01:27,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:29,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:30,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:31,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:32,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:33,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:35,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:36,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:37,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:37,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:39,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:41,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:01:41,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:41,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:43,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:45,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:47,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:47,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:47,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:49,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:50,843 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:01:51,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:52,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:01:52,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:53,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:55,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:56,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:01:57,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:01:57,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:01:59,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:01,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:01,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:03,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:03,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:02:03,365 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:02:03,366 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:02:03,367 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:02:03,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:03,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:02:05,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:06,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:06,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:07,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:08,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:09,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:11,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:02:11,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:13,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:14,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:15,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:16,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:17,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:19,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:19,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:20,863 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:02:21,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:21,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:23,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:24,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:25,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:26,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:02:27,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:29,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:30,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:31,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:31,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:33,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:35,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:35,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:36,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:37,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:39,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:41,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:02:41,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:41,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:43,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:45,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:46,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:46,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:47,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:48,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:02:48,313 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:02:48,313 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:02:48,315 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:02:48,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:02:49,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:50,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:50,847 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:02:51,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:02:51,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:51,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:53,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:55,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:56,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:02:57,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:02:57,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:02:59,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:01,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:01,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:02,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:03,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:05,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:06,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:07,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:09,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:11,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:03:11,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:13,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:13,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:15,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:16,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:17,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:18,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:19,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:20,850 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:03:21,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:21,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:23,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:24,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:25,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:26,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:03:27,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:29,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:29,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:31,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:31,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:33,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:03:33,364 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:03:33,365 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:03:33,367 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:03:33,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:33,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:03:35,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:35,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:36,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:36,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:37,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:39,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:40,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:41,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:03:41,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:43,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:45,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:45,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:46,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:47,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:49,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:50,852 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:03:51,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:03:51,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:51,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:53,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:55,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:56,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:03:56,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:03:57,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:03:59,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:01,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:01,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:02,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:03,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:05,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:06,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:07,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:07,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:09,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:11,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:04:11,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:12,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:13,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:15,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:16,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:17,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:18,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:18,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:04:18,372 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:04:18,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:04:18,374 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:04:18,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:04:19,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:20,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:20,864 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:04:21,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:21,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:23,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:23,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:25,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:26,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:04:27,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:29,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:29,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:31,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:31,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:33,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:34,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:35,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:36,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:37,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:39,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:39,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:41,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:04:41,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:43,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:45,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:45,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:46,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:47,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:49,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:50,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:50,857 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:04:51,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:04:51,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:53,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:55,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:56,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:04:56,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:04:57,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:04:59,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:01,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:01,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:01,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:03,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:05:03,377 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:05:03,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:05:03,379 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:05:03,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:03,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:05:05,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:06,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:06,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:06,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:07,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:09,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:11,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:05:11,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:12,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:13,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:15,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:16,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:17,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:17,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:19,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:20,859 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:05:21,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:21,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:23,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:23,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:25,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:26,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:05:27,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:28,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:29,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:31,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:31,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:33,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:33,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:36,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:37,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:39,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:39,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:41,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:05:41,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:43,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:44,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:45,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:46,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:47,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:48,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:05:48,311 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:05:48,311 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:05:48,313 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:05:48,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:05:49,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:50,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:50,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:50,861 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:05:51,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:05:51,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:53,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:55,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:05:55,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:56,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:05:56,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:05:58,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:00,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:00,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:01,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:02,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:04,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:06,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:06,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:06,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:08,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:10,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:11,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:06:11,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:12,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:14,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:16,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:16,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:17,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:18,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:20,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:20,873 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:06:21,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:22,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:22,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:24,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:26,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:06:26,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:27,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:28,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:30,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:31,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:32,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:33,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:33,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:06:33,192 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:06:33,192 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:06:33,195 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:06:33,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:06:34,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:36,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:36,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:38,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:38,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:40,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:41,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:06:42,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:43,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:44,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:46,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:46,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:48,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:49,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:50,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:50,866 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:06:51,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:06:52,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:54,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:06:54,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:56,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:06:56,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:06:58,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:00,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:00,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:01,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:02,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:04,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:05,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:06,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:06,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:08,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:10,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:10,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:11,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:07:12,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:14,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:16,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:16,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:16,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:18,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:07:18,175 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:07:18,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:07:18,177 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:07:18,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:18,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:07:19,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:20,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:20,868 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:07:21,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:21,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:22,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:24,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:26,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:07:26,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:27,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:28,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:30,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:31,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:32,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:32,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:34,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:36,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:36,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:37,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:38,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:40,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:41,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:07:42,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:43,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:44,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:46,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:46,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:48,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:48,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:50,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:50,870 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:07:51,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:07:52,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:54,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:07:54,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:56,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:07:56,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:58,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:07:59,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:00,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:01,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:02,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:03,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:08:03,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:08:03,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:08:03,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:08:03,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:03,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:08:04,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:04,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:06,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:06,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:08,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:10,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:10,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:11,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:08:12,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:14,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:15,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:16,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:16,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:18,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:20,888 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:08:20,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:20,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:21,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:22,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:24,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:26,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:08:26,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:26,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:28,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:30,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:31,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:31,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:32,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:34,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:36,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:36,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:37,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:38,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:40,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:41,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:08:42,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:42,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:44,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:46,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:46,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:48,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:48,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:08:48,192 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:08:48,193 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:08:48,194 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:08:48,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:48,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:08:50,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:50,875 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:08:51,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:51,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:08:53,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:53,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:55,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:56,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:08:57,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:08:58,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:08:59,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:01,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:01,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:03,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:04,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:05,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:06,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:09,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:11,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:09:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:13,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:15,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:15,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:16,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:17,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:19,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:20,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:20,877 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:09:21,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:21,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:23,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:25,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:25,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:26,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:09:27,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:29,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:31,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:31,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:31,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:33,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:09:33,199 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:09:33,199 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:09:33,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:33,201 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:09:34,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:34,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:09:35,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:36,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:36,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:37,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:39,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:41,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:09:41,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:42,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:43,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:45,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:46,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:47,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:47,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:49,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:50,879 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:09:51,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:51,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:09:52,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:53,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:55,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:56,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:09:57,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:09:58,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:09:59,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:01,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:01,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:03,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:03,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:05,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:06,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:07,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:09,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:09,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:11,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:10:11,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:13,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:14,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:15,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:16,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:17,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:18,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:10:18,190 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:10:18,190 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:10:18,192 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:10:18,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:10:19,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:19,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:20,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:20,892 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:10:21,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:21,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:23,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:25,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:25,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:26,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:10:27,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:29,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:30,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:31,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:31,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:33,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:35,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:36,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:36,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:37,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:39,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:41,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:10:41,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:41,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:43,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:45,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:46,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:46,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:47,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:49,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:50,884 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:10:51,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:51,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:10:52,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:53,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:55,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:56,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:10:57,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:10:57,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:10:59,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:01,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:01,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:03,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:03,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:11:03,109 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:11:03,109 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:11:03,111 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:11:03,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:03,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:11:04,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:05,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:06,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:07,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:08,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:09,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:11,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:11:11,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:13,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:13,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:15,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:16,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:17,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:19,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:19,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:20,886 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:11:21,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:21,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:23,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:24,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:25,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:26,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:11:27,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:29,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:30,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:31,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:31,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:33,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:35,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:35,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:36,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:37,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:39,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:40,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:11:41,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:43,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:45,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:46,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:46,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:47,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:48,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:11:48,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:11:48,111 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:11:48,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:11:48,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:11:49,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:50,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:50,887 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:11:51,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:51,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:11:51,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:53,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:55,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:56,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:11:57,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:11:57,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:11:59,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:01,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:01,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:02,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:03,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:05,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:06,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:07,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:07,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:09,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:11,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:12:11,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:13,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:13,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:15,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:16,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:17,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:18,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:19,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:20,901 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:12:21,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:21,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:23,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:24,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:25,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:26,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:12:27,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:29,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:29,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:31,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:31,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:33,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:12:33,101 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:12:33,102 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:12:33,103 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:12:33,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:33,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:12:34,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:34,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:35,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:36,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:37,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:39,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:12:41,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:43,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:45,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:45,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:46,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:47,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:49,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:50,892 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:12:50,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:51,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:12:51,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:53,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:55,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:56,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:12:56,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:12:57,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:12:59,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:01,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:01,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:01,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:03,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:05,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:06,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:07,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:07,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:09,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:11,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:13:11,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:12,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:13,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:15,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:16,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:17,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:18,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:18,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:13:18,074 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:13:18,075 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:13:18,076 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:13:18,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:13:19,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:20,895 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:13:21,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:21,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:23,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:23,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:25,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:26,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:13:27,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:28,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:29,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:31,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:31,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:33,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:34,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:35,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:36,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:37,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:39,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:39,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:13:41,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:43,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:45,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:45,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:46,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:47,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:49,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:50,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:50,897 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:13:51,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:13:51,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:53,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:55,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:55,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:13:56,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:13:57,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:13:59,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:01,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:01,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:01,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:03,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:14:03,080 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:14:03,080 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:14:03,082 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:14:03,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:03,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:14:04,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:05,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:06,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:06,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:07,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:09,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:11,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:14:11,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:12,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:13,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:15,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:16,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:17,450 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:17,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:19,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:20,924 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:14:21,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:21,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:22,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:23,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:25,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:26,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:14:27,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:28,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:29,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:31,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:31,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:33,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:33,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:35,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:36,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:37,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:39,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:39,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:41,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:14:41,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:43,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:44,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:45,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:46,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:47,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:48,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:14:48,085 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:14:48,086 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:14:48,087 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:14:48,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:14:49,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:49,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:50,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:50,901 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:14:51,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:14:51,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:53,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:55,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:14:55,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:56,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:14:57,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:14:59,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:00,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:01,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:01,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:03,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:05,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:06,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:06,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:07,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:09,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:11,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:15:11,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:11,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:13,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:15,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:16,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:16,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:17,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:19,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:20,903 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:15:21,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:21,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:21,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:23,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:25,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:26,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:15:27,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:27,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:29,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:31,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:31,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:32,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:33,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:15:33,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:15:33,025 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:15:33,026 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:15:33,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:33,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:15:34,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:35,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:36,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:37,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:38,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:39,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:41,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:15:41,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:43,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:43,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:45,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:46,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:47,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:49,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:49,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:50,906 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:15:51,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:15:51,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:53,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:54,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:15:55,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:56,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:15:57,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:59,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:15:59,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:01,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:01,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:03,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:05,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:05,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:06,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:07,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:09,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:10,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:11,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:16:11,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:13,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:15,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:16,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:16,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:17,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:17,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:16:17,961 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:16:17,961 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:16:17,963 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:16:18,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:16:19,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:20,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:20,918 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:16:21,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:21,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:21,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:23,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:25,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:26,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:16:26,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:27,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:29,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:31,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:31,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:32,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:33,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:35,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:36,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:37,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:37,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:39,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:41,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:16:41,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:43,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:43,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:45,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:46,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:47,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:48,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:49,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:50,910 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:16:51,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:16:51,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:53,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:53,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:55,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:56,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:16:57,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:16:59,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:16:59,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:01,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:01,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:02,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:17:02,927 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:17:02,927 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:17:02,930 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:17:03,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:03,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:17:04,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:17:04,497 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:17:04,499 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:17:04,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:04,501 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:17:04,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:04,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:17:05,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:06,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:07,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:08,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:09,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:10,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:11,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:17:11,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:15,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:16,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:19,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:20,912 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:17:20,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:21,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:21,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:23,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:25,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:26,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:17:26,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:27,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:29,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:31,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:31,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:31,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:33,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:35,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:36,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:37,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:37,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:39,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:41,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:17:41,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:42,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:43,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:45,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:46,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:47,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:48,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:49,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:50,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:17:51,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:51,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:17:53,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:53,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:55,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:56,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:17:57,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:17:58,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:17:59,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:01,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:01,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:03,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:18:03,399 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:18:03,399 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:18:03,401 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:18:03,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:03,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:18:04,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:04,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:05,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:06,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:07,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:09,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:09,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:11,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:18:11,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:13,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:15,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:15,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:16,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:17,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:19,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:20,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:20,918 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:18:21,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:21,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:23,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:25,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:25,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:26,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:18:27,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:29,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:31,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:31,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:31,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:33,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:35,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:36,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:36,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:37,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:39,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:41,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:18:41,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:42,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:43,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:45,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:46,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:47,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:47,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:48,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:18:48,381 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:18:48,382 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:18:48,383 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:18:48,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:18:49,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:50,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:50,920 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:18:51,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:18:51,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:52,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:53,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:55,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:56,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:18:57,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:18:58,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:18:59,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:01,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:01,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:03,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:03,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:05,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:06,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:06,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:08,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:09,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:10,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:11,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:19:12,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:14,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:15,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:16,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:17,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:19,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:19,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:20,923 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:19:21,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:21,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:23,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:25,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:25,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:26,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:19:27,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:29,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:30,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:31,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:31,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:33,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:33,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:19:33,402 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:19:33,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:19:33,404 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:19:34,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:34,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:19:35,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:36,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:37,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:39,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:41,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:41,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:19:41,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:43,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:45,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:46,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:46,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:47,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:49,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:50,925 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:19:51,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:51,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:19:52,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:53,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:55,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:56,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:19:57,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:19:57,835 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:19:59,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:01,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:01,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:03,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:03,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:05,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:06,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:07,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:08,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:09,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:11,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:11,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:20:13,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:14,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:15,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:16,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:17,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:18,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:20:18,477 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:20:18,478 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:20:18,480 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:20:19,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:19,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:20:19,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:20,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:20,927 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:20:21,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:21,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:23,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:24,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:25,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:26,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:20:27,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:29,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:30,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:31,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:31,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:33,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:35,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:35,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:36,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:37,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:39,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:40,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:41,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:20:41,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:43,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:45,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:46,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:46,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:47,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:49,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:50,929 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:20:51,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:51,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:51,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:20:53,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:55,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:56,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:20:57,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:20:57,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:20:59,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:01,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:01,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:02,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:03,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:03,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:21:03,431 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:21:03,432 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:21:03,433 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:21:04,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:04,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:21:05,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:06,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:07,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:07,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:09,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:11,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:21:11,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:13,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:13,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:15,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:16,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:17,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:18,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:19,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:20,931 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:21:21,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:21,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:23,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:24,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:25,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:26,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:21:27,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:29,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:29,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:31,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:31,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:33,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:34,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:35,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:36,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:37,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:39,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:40,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:41,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:21:41,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:43,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:45,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:45,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:47,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:48,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:21:48,333 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:21:48,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:21:48,335 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:21:49,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:49,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:21:50,976 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:21:50,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:51,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:52,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:21:53,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:55,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:56,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:21:56,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:21:57,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:21:59,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:01,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:01,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:02,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:03,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:05,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:07,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:07,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:07,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:09,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:11,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:22:11,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:12,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:13,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:15,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:17,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:17,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:18,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:19,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:20,979 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:22:21,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:22,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:23,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:23,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:25,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:22:27,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:28,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:29,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:31,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:32,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:33,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:22:33,273 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:22:33,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:22:33,276 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:22:33,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:33,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:22:34,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:35,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:37,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:37,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:39,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:39,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:41,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:22:41,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:43,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:45,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:45,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:47,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:47,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:49,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:50,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:50,981 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:22:51,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:52,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:22:53,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:55,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:22:56,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:22:57,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:22:59,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:01,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:01,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:02,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:03,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:05,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:06,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:07,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:07,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:09,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:11,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:23:11,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:12,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:13,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:15,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:17,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:17,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:17,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:18,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:23:18,298 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:23:18,298 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:23:18,300 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:23:18,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:23:19,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:20,983 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:23:21,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:22,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:23,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:25,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:23:27,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:28,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:29,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:31,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:32,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:33,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:33,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:35,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:37,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:37,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:39,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:39,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:41,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:23:41,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:43,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:44,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:45,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:47,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:47,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:49,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:49,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:50,985 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:23:51,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:52,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:23:53,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:55,139 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:23:55,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:56,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:23:57,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:23:59,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:00,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:01,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:02,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:03,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:24:03,135 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:24:03,135 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:24:03,137 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:24:03,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:03,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:24:05,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:05,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:07,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:07,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:09,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:11,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:24:11,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:11,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:13,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:15,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:16,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:17,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:17,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:19,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:20,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:24:21,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:22,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:22,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:23,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:25,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:24:27,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:27,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:29,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:31,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:32,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:32,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:33,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:35,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:37,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:37,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:38,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:39,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:41,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:24:41,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:43,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:43,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:45,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:47,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:47,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:48,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:24:48,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:24:48,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:24:48,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:24:48,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:24:49,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:49,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:50,989 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:24:51,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:52,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:24:53,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:54,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:24:55,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:56,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:24:57,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:59,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:24:59,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:01,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:02,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:03,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:05,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:05,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:07,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:07,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:09,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:10,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:11,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:25:11,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:13,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:15,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:16,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:17,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:17,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:19,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:20,991 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:25:21,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:21,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:22,096 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:23,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:25,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:26,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:25:27,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:27,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:29,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:31,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:32,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:33,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:33,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:25:33,144 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:25:33,145 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:25:33,147 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:25:33,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:33,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:25:35,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:37,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:37,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:38,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:39,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:41,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:25:41,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:43,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:43,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:45,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:47,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:49,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:49,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:50,993 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:25:51,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:52,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:25:53,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:54,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:25:55,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:56,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:25:57,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:25:59,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:00,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:01,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:02,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:03,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:05,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:05,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:07,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:07,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:09,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:10,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:11,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:26:11,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:13,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:15,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:16,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:17,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:17,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:18,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:26:18,086 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:26:18,086 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:26:18,088 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:26:18,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:26:19,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:21,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:26:21,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:21,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:22,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:23,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:25,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:26,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:26:27,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:27,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:29,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:31,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:32,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:32,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:33,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:35,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:37,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:37,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:37,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:39,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:41,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:26:41,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:43,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:43,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:45,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:46,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:47,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:48,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:49,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:50,997 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:26:51,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:26:51,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:53,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:54,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:55,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:56,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:26:57,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:26:59,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:26:59,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:01,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:01,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:03,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:27:03,088 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:27:03,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:27:03,091 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:27:03,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:03,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:27:04,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:05,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:06,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:07,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:09,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:10,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:11,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:27:11,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:13,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:15,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:15,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:16,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:17,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:19,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:20,999 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:27:21,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:21,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:21,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:23,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:25,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:26,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:27:26,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:27,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:29,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:31,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:31,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:31,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:33,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:35,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:36,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:37,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:37,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:39,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:41,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:27:41,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:42,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:43,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:45,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:46,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:47,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:48,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:48,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:27:48,103 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:27:48,103 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:27:48,106 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:27:48,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:27:49,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:50,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:51,002 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:27:51,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:27:51,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:53,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:53,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:55,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:56,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:27:57,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:27:58,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:27:59,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:01,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:01,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:03,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:04,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:05,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:06,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:07,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:09,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:09,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:11,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:28:11,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:13,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:15,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:15,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:16,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:17,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:19,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:20,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:21,025 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:28:21,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:21,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:23,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:25,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:25,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:26,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:28:27,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:29,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:31,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:31,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:31,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:33,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:28:33,062 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:28:33,062 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:28:33,064 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:28:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:28:34,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:35,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:36,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:36,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:37,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:39,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:41,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:28:41,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:42,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:43,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:45,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:46,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:47,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:47,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:49,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:51,007 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:28:51,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:28:51,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:53,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:55,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:56,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:28:57,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:28:58,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:28:59,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:01,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:01,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:03,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:03,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:05,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:06,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:07,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:09,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:09,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:11,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:29:11,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:13,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:14,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:15,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:16,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:17,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:18,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:29:18,167 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:29:18,167 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:29:18,168 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:29:18,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:29:19,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:19,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:20,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:21,009 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:29:21,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:21,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:23,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:25,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:26,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:29:27,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:29,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:30,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:31,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:31,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:33,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:36,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:36,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:37,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:39,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:41,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:29:41,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:41,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:43,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:46,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:46,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:46,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:48,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:50,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:51,011 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:29:51,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:29:52,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:52,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:54,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:56,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:29:56,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:29:57,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:29:58,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:00,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:01,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:02,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:03,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:03,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:30:03,089 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:30:03,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:30:03,092 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:30:04,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:04,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:30:06,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:06,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:08,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:08,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:10,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:11,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:30:12,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:13,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:14,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:16,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:16,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:18,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:19,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:20,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:21,023 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:30:21,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:22,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:24,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:24,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:26,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:26,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:30:28,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:30,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:30,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:31,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:32,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:34,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:35,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:36,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:36,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:38,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:40,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:40,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:41,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:30:42,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:44,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:46,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:46,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:46,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:47,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:30:47,989 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:30:47,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:30:47,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:30:48,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:48,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:30:50,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:51,015 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:30:51,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:30:51,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:52,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:54,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:56,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:30:56,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:30:56,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:30:58,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:00,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:01,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:02,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:02,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:04,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:06,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:06,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:07,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:08,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:10,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:31:12,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:13,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:14,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:16,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:16,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:18,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:18,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:20,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:21,017 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:31:21,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:22,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:23,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:24,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:26,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:26,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:31:28,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:29,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:30,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:31,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:32,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:32,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:31:32,892 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:31:32,892 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:31:32,894 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:31:33,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:31:34,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:34,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:36,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:36,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:38,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:40,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:40,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:41,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:31:42,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:44,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:45,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:46,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:46,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:48,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:50,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:50,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:51,020 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:31:51,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:31:52,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:54,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:56,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:31:56,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:31:56,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:31:58,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:00,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:01,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:01,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:02,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:04,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:06,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:06,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:07,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:08,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:10,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:32:12,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:12,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:14,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:16,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:16,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:17,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:17,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:32:17,816 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:32:17,816 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:32:17,818 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:32:18,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:18,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:32:20,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:21,022 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:32:21,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:22,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:23,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:24,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:26,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:32:26,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:28,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:28,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:30,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:31,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:32,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:33,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:34,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:36,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:36,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:37,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:39,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:39,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:41,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:32:41,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:43,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:44,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:45,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:46,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:47,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:49,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:50,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:51,024 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:32:51,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:51,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:32:53,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:55,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:55,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:32:56,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:32:57,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:32:59,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:00,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:01,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:01,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:02,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:33:02,699 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:33:02,700 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:33:02,701 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:33:03,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:03,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:33:04,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:05,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:06,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:06,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:07,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:09,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:33:11,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:11,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:13,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:15,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:16,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:17,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:17,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:19,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:21,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:33:21,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:21,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:22,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:23,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:25,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:26,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:33:27,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:27,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:29,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:31,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:31,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:33,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:33,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:35,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:36,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:37,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:38,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:39,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:41,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:33:41,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:43,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:43,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:45,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:46,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:47,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:47,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:33:47,540 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:33:47,540 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:33:47,542 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:33:48,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:48,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:33:49,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:49,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:51,029 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:33:51,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:51,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:33:53,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:54,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:33:55,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:56,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:33:57,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:33:59,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:00,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:01,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:01,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:03,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:05,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:05,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:06,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:07,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:09,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:10,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:11,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:34:11,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:13,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:15,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:16,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:16,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:17,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:19,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:21,031 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:34:21,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:21,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:21,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:23,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:25,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:26,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:34:27,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:27,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:29,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:31,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:31,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:32,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:32,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:34:32,490 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:34:32,491 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:34:32,492 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:34:33,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:33,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:34:34,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:35,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:36,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:37,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:37,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:39,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:41,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:34:41,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:43,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:45,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:46,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:47,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:48,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:49,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:51,033 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:34:51,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:51,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:34:53,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:53,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:34:55,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:56,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:34:57,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:59,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:34:59,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:01,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:01,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:03,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:04,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:05,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:06,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:07,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:09,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:10,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:11,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:35:11,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:13,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:15,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:15,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:16,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:17,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:35:17,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:35:17,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:35:17,394 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:35:17,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:17,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:35:18,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:19,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:20,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:21,035 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:35:21,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:21,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:23,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:25,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:26,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:35:26,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:27,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:29,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:31,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:31,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:31,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:33,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:35,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:36,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:37,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:37,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:39,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:41,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:35:41,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:42,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:43,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:45,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:46,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:47,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:47,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:49,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:51,037 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:35:51,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:51,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:35:53,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:53,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:55,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:56,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:35:57,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:35:58,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:35:59,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:01,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:01,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:02,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:36:02,336 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:36:02,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:36:02,338 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:36:02,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:36:03,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:04,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:04,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:05,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:06,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:07,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:09,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:09,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:11,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:36:11,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:13,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:14,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:15,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:16,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:17,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:19,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:20,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:21,040 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:36:21,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:21,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:23,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:25,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:25,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:26,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:36:27,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:29,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:30,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:31,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:31,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:33,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:35,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:36,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:36,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:37,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:39,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:41,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:36:41,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:41,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:43,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:45,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:46,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:47,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:47,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:36:47,205 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:36:47,205 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:36:47,207 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:36:47,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:47,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:36:48,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:49,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:51,041 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:36:51,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:36:51,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:52,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:53,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:55,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:56,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:36:57,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:36:57,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:36:59,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:01,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:01,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:03,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:03,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:05,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:06,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:07,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:08,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:09,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:11,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:37:11,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:13,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:14,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:15,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:16,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:17,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:19,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:19,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:21,043 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:37:21,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:21,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:23,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:24,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:25,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:26,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:37:27,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:29,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:30,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:31,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:31,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:32,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:37:32,198 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:37:32,198 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:37:32,200 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:37:32,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:37:33,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:34,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:35,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:35,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:36,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:37,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:39,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:41,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:41,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:37:41,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:43,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:45,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:46,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:46,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:47,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:49,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:51,046 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:37:51,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:37:51,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:51,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:53,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:55,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:56,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:37:57,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:37:57,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:37:59,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:01,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:01,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:02,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:03,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:05,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:06,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:07,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:08,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:09,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:38:11,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:13,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:13,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:15,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:16,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:17,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:38:17,092 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:38:17,093 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:38:17,095 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:38:17,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:17,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:38:18,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:18,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:19,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:21,060 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:38:21,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:21,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:23,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:24,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:25,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:26,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:38:27,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:29,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:29,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:31,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:31,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:33,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:34,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:35,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:36,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:37,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:39,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:41,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:38:41,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:43,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:45,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:45,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:46,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:47,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:49,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:51,050 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:38:51,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:51,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:38:51,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:53,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:55,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:56,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:38:56,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:38:57,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:38:59,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:01,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:01,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:01,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:01,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:39:01,945 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:39:01,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:39:01,947 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:39:02,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:39:03,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:04,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:05,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:06,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:07,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:07,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:09,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:39:11,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:12,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:13,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:15,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:16,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:17,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:18,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:19,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:21,052 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:39:21,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:21,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:23,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:23,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:25,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:26,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:39:27,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:28,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:29,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:31,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:31,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:33,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:34,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:35,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:36,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:37,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:39,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:39,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:41,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:39:41,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:43,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:45,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:46,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:46,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:39:46,857 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:39:46,857 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:39:46,859 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:39:46,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:39:47,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:48,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:49,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:50,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:51,055 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:39:51,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:39:51,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:53,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:55,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:39:55,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:39:56,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:39:57,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:00,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:01,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:01,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:02,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:04,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:06,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:06,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:06,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:08,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:10,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:40:11,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:12,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:14,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:16,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:16,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:17,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:18,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:20,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:21,067 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:40:21,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:22,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:22,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:24,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:26,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:26,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:40:28,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:28,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:30,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:31,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:31,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:40:31,725 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:40:31,725 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:40:31,728 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:40:32,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:32,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:40:33,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:33,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:34,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:36,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:36,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:38,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:38,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:40,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:41,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:40:42,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:44,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:44,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:46,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:46,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:48,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:49,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:50,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:51,059 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:40:51,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:40:52,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:54,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:54,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:40:56,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:40:56,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:40:58,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:00,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:00,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:01,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:02,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:04,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:05,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:06,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:06,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:08,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:10,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:11,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:11,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:41:12,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:14,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:16,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:16,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:16,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:41:16,607 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:41:16,607 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:41:16,610 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:41:16,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:17,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:41:18,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:19,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:20,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:21,061 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:41:21,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:21,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:22,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:24,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:26,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:26,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:41:27,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:28,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:30,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:31,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:32,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:32,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:34,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:36,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:36,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:38,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:38,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:40,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:41,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:41:42,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:43,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:44,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:46,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:46,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:48,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:48,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:50,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:51,063 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:41:51,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:41:52,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:54,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:54,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:41:56,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:56,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:41:58,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:41:59,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:00,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:01,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:42:01,498 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:42:01,498 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:42:01,501 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:42:01,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:02,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:02,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:42:03,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:04,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:05,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:06,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:06,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:08,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:10,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:10,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:11,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:42:12,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:14,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:15,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:16,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:16,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:18,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:20,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:21,075 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:42:21,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:21,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:22,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:24,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:26,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:26,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:42:26,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:28,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:30,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:31,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:31,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:32,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:34,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:36,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:36,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:37,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:38,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:40,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:41,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:42:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:42,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:44,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:46,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:46,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:42:46,336 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:42:46,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:42:46,338 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:42:46,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:47,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:42:48,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:48,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:50,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:51,068 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:42:51,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:42:52,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:53,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:42:54,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:56,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:56,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:42:58,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:42:58,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:00,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:01,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:02,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:04,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:04,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:06,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:06,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:08,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:09,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:10,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:11,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:43:12,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:14,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:14,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:16,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:16,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:18,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:20,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:20,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:21,070 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:43:21,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:22,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:24,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:25,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:26,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:43:26,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:28,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:30,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:31,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:31,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:43:31,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:43:31,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:43:31,250 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:43:31,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:43:31,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:32,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:34,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:36,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:36,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:36,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:38,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:40,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:41,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:43:42,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:42,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:44,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:46,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:46,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:47,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:48,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:50,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:51,072 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:43:51,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:43:52,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:52,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:54,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:56,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:43:56,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:43:58,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:43:58,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:00,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:01,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:02,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:03,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:04,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:06,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:06,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:08,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:08,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:10,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:11,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:44:12,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:14,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:14,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:16,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:44:16,102 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:44:16,102 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:44:16,104 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:44:16,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:16,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:44:16,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:18,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:19,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:20,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:21,085 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:44:21,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:22,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:24,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:25,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:26,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:44:26,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:28,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:30,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:30,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:31,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:32,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:34,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:35,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:36,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:36,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:38,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:40,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:41,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:41,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:44:42,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:44,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:46,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:46,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:46,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:48,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:50,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:51,077 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:44:51,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:44:51,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:52,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:54,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:56,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:44:56,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:44:57,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:44:58,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:00,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:00,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:45:00,973 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:45:00,973 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:45:00,976 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:45:01,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:45:01,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:02,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:02,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:04,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:06,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:06,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:08,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:08,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:10,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:11,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:45:12,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:13,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:14,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:16,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:16,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:18,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:18,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:20,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:21,078 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:45:21,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:22,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:24,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:24,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:26,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:45:26,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:28,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:29,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:30,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:31,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:32,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:34,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:34,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:36,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:40,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:40,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:41,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:45:42,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:44,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:45,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:45,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:45:45,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:45:45,691 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:45:45,694 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:45:46,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:46,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:45:46,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:48,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:50,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:51,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:51,081 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:45:51,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:45:52,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:54,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:56,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:45:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:45:56,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:45:58,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:00,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:01,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:01,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:02,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:04,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:06,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:06,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:07,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:08,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:10,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:11,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:46:12,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:12,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:14,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:16,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:16,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:17,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:18,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:20,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:21,097 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:46:21,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:22,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:23,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:24,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:26,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:46:26,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:28,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:28,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:30,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:30,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:46:30,559 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:46:30,559 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:46:30,561 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:46:31,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:46:31,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:32,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:33,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:34,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:35,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:36,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:37,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:39,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:39,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:41,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:46:41,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:43,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:44,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:45,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:46,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:49,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:50,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:51,085 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:46:51,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:51,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:46:53,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:55,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:55,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:46:56,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:46:57,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:46:59,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:01,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:01,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:01,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:03,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:05,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:06,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:06,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:07,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:09,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:11,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:47:11,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:12,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:13,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:15,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:15,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:47:15,620 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:47:15,620 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:47:15,622 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:47:16,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:16,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:47:16,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:17,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:17,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:19,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:21,087 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:47:21,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:21,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:22,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:23,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:25,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:26,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:47:27,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:28,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:29,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:31,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:31,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:33,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:33,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:35,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:36,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:37,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:38,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:39,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:41,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:47:41,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:43,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:44,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:45,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:46,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:47,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:49,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:49,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:51,089 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:47:51,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:51,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:47:53,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:55,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:47:55,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:56,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:47:57,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:47:59,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:00,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:00,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:48:00,497 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:48:00,497 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:48:00,499 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:48:00,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:48:01,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:01,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:02,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:03,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:05,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:05,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:06,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:07,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:09,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:11,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:11,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:48:11,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:13,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:15,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:16,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:16,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:17,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:19,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:21,094 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:48:21,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:21,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:22,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:23,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:25,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:26,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:48:27,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:27,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:29,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:31,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:31,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:32,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:33,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:35,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:36,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:37,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:39,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:41,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:48:41,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:43,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:43,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:45,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:48:45,411 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:48:45,412 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:48:45,414 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:48:45,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:45,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:48:46,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:46,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:47,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:48,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:49,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:51,096 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:48:51,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:51,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:48:53,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:54,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:55,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:56,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:48:57,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:48:59,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:48:59,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:01,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:01,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:03,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:05,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:05,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:06,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:07,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:09,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:10,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:11,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:49:11,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:13,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:15,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:15,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:17,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:17,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:19,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:21,098 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:49:21,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:21,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:22,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:23,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:25,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:26,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:49:26,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:27,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:29,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:30,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:49:30,238 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:49:30,238 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:49:30,240 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:49:30,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:49:31,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:32,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:32,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:32,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:33,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:35,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:37,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:37,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:37,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:39,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:41,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:49:41,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:42,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:43,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:45,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:47,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:47,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:48,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:49,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:51,100 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:49:51,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:52,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:49:53,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:49:53,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:55,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:56,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:49:58,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:49:58,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:00,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:02,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:02,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:04,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:04,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:06,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:07,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:08,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:09,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:10,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:11,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:50:12,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:14,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:14,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:14,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:50:14,984 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:50:14,985 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:50:14,986 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:50:15,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:50:16,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:17,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:17,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:18,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:20,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:20,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:21,113 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:50:22,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:22,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:24,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:25,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:26,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:26,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:50:28,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:30,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:31,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:32,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:32,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:34,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:36,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:36,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:37,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:38,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:40,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:41,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:50:41,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:42,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:44,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:46,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:47,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:47,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:48,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:50,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:51,105 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:50:52,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:52,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:50:52,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:54,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:56,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:56,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:50:58,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:50:58,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:50:59,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:50:59,875 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:50:59,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:50:59,877 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:51:00,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:00,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:51:02,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:02,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:03,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:03,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:04,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:06,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:07,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:08,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:08,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:10,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:11,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:51:12,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:14,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:14,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:16,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:17,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:18,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:19,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:20,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:21,107 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:51:22,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:22,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:24,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:24,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:26,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:26,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:51:28,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:30,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:30,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:32,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:32,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:34,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:35,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:36,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:37,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:38,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:40,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:41,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:41,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:51:42,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:44,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:44,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:51:44,705 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:51:44,706 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:51:44,708 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:51:45,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:51:46,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:46,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:47,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:47,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:48,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:50,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:51,109 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:51:51,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:52,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:52,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:51:54,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:56,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:51:56,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:51:57,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:51:58,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:00,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:02,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:02,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:02,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:04,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:06,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:07,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:07,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:08,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:10,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:11,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:52:12,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:13,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:14,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:16,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:17,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:18,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:18,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:20,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:21,128 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:52:22,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:22,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:24,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:24,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:26,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:26,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:52:28,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:29,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:29,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:52:29,523 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:52:29,523 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:52:29,525 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:52:30,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:30,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:52:31,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:32,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:32,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:34,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:34,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:36,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:37,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:38,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:40,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:40,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:41,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:52:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:44,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:45,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:46,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:47,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:48,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:50,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:50,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:51,113 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:52:52,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:52,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:52:54,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:56,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:52:56,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:52:56,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:52:58,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:00,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:01,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:02,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:02,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:04,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:06,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:06,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:07,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:08,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:10,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:11,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:53:12,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:12,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:13,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:53:13,993 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:53:13,994 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:53:13,996 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:53:14,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:14,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:53:16,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:17,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:17,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:17,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:18,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:20,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:21,115 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:53:22,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:22,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:22,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:24,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:26,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:26,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:53:28,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:28,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:30,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:32,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:32,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:33,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:34,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:36,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:37,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:38,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:38,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:40,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:41,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:53:42,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:44,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:44,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:46,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:47,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:48,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:49,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:50,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:51,118 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:53:52,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:53:52,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:54,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:54,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:53:56,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:53:56,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:58,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:53:58,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:53:58,478 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:53:58,479 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:53:58,481 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:53:59,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:54:00,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:00,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:02,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:02,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:04,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:05,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:06,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:07,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:08,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:10,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:10,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:11,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:54:12,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:14,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:16,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:16,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:17,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:18,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:20,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:21,130 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:54:21,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:22,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:22,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:24,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:26,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:54:26,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:26,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:28,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:30,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:32,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:32,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:32,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:34,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:36,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:37,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:37,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:38,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:40,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:41,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:54:42,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:42,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:42,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:54:42,995 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:54:42,995 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:54:42,998 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:54:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:54:44,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:46,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:47,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:48,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:48,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:50,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:51,122 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:54:52,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:54:52,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:53,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:54:54,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:56,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:54:56,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:58,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:54:59,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:00,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:02,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:02,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:04,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:04,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:06,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:07,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:08,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:09,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:10,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:11,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:55:12,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:14,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:15,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:16,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:16,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:18,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:20,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:20,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:21,124 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:55:21,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:22,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:24,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:25,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:26,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:55:26,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:27,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:55:27,487 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:55:27,487 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:55:27,489 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:55:28,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:28,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:55:30,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:30,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:31,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:32,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:34,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:36,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:36,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:36,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:38,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:40,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:41,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:55:41,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:42,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:44,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:46,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:46,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:46,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:48,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:50,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:51,126 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:55:51,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:55:52,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:52,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:54,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:56,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:55:56,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:55:57,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:55:58,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:00,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:01,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:02,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:03,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:04,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:06,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:06,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:08,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:10,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:11,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:56:11,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:56:11,939 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:56:11,939 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:56:11,942 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:56:12,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:12,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:56:13,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:14,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:16,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:16,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:18,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:18,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:21,140 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:56:21,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:22,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:24,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:24,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:26,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:56:26,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:28,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:29,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:30,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:31,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:32,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:34,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:34,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:36,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:36,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:38,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:40,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:40,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:41,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:56:42,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:44,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:45,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:46,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:46,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:48,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:50,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:50,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:51,131 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:56:51,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:56:52,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:54,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:56,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:56:56,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:56:56,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:56:56,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:56:56,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:56:56,394 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:56:56,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:56:56,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:56:58,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:00,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:01,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:01,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:02,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:04,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:06,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:06,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:07,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:08,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:10,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:11,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:57:12,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:12,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:14,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:16,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:16,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:17,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:18,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:20,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:21,133 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:57:21,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:22,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:22,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:24,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:26,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:57:26,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:28,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:28,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:31,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:32,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:33,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:34,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:36,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:36,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:38,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:38,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:40,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:40,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:57:40,788 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:57:40,789 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:57:40,791 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:57:41,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:57:41,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:57:42,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:44,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:44,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:46,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:46,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:48,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:49,692 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:50,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:51,135 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:57:51,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:57:52,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:54,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:55,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:57:56,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:57:56,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:57:58,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:00,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:00,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:01,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:02,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:04,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:05,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:06,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:06,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:08,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:10,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:11,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:11,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:58:12,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:14,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:16,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:16,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:16,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:18,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:20,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:21,148 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:58:21,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:21,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:22,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:24,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:58:25,284 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:58:25,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:58:25,287 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:58:25,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:58:26,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:58:26,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:27,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:28,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:30,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:31,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:32,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:32,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:34,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:36,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:36,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:37,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:38,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:40,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:41,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:58:42,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:42,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:43,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:45,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:46,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:47,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:48,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:49,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:51,139 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:58:51,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:58:51,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:53,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:53,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:55,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:56,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:58:57,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:58:59,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:58:59,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:01,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:01,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:03,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:04,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:05,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:06,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:07,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:09,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:09,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:09,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:59:09,893 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:59:09,893 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:59:09,897 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:59:10,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:10,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:59:11,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:59:11,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:13,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:15,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:15,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:16,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:17,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:19,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:20,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:21,142 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:59:21,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:21,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:23,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:25,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:25,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:26,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:59:27,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:29,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:31,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:31,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:31,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:33,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:36,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:36,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:37,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:39,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:41,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:59:41,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:41,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:43,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:46,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:46,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:47,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:48,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:50,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:51,144 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 21:59:51,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 21:59:52,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:52,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:54,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:54,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 21:59:54,383 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 21:59:54,384 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 21:59:54,386 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 21:59:55,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 21:59:56,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:56,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 21:59:57,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 21:59:57,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 21:59:58,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:00,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:01,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:02,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:03,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:04,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:06,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:06,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:08,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:08,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:10,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:11,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:00:12,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:13,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:14,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:16,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:16,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:18,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:19,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:20,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:21,146 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:00:21,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:22,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:24,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:24,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:26,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:26,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:00:28,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:29,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:30,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:31,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:32,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:34,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:35,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:36,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:36,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:38,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:38,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:00:38,791 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:00:38,792 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:00:38,793 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:00:39,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:00:40,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:40,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:41,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:41,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:00:42,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:44,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:45,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:46,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:46,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:48,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:50,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:51,148 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:00:51,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:51,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:00:52,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:54,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:56,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:00:56,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:00:56,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:00:58,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:00,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:01,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:01,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:02,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:04,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:06,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:06,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:07,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:08,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:10,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:11,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:01:12,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:12,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:14,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:16,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:16,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:17,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:18,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:20,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:21,151 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:01:21,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:22,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:23,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:23,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:01:23,312 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:01:23,313 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:01:23,314 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:01:24,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:24,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:01:25,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:26,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:26,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:01:28,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:28,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:30,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:31,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:32,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:33,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:34,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:36,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:36,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:38,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:39,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:40,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:41,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:01:42,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:44,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:44,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:46,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:46,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:48,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:49,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:50,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:51,152 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:01:51,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:01:52,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:54,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:55,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:01:56,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:01:56,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:01:58,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:00,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:00,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:01,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:02,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:04,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:05,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:06,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:06,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:07,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:02:07,776 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:02:07,777 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:02:07,778 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:02:08,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:08,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:02:09,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:10,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:11,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:02:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:12,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:14,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:16,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:16,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:16,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:18,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:20,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:21,166 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:02:21,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:21,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:22,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:24,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:26,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:26,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:02:27,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:28,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:30,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:31,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:32,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:32,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:34,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:36,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:36,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:37,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:38,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:40,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:41,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:02:42,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:43,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:44,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:46,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:46,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:48,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:48,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:50,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:51,156 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:02:51,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:02:52,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:52,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:02:52,272 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:02:52,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:02:52,275 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:02:53,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:02:54,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:02:54,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:55,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:56,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:56,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:02:58,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:02:59,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:00,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:01,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:02,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:04,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:04,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:06,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:06,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:08,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:10,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:10,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:11,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:03:12,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:14,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:15,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:16,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:16,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:18,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:20,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:20,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:21,159 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:03:21,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:22,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:24,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:26,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:26,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:03:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:28,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:30,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:31,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:31,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:32,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:34,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:36,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:36,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:36,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:37,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:03:37,021 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:03:37,021 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:03:37,023 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:03:37,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:03:38,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:39,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:40,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:41,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:03:42,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:42,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:44,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:46,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:46,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:47,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:48,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:50,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:51,161 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:03:51,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:03:52,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:53,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:03:54,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:56,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:03:56,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:58,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:03:58,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:00,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:01,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:02,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:03,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:04,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:06,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:06,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:08,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:09,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:10,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:11,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:04:12,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:14,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:14,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:16,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:16,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:18,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:19,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:20,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:21,174 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:04:21,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:04:21,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:04:21,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:04:21,693 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:04:21,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:22,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:22,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:04:23,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:24,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:25,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:26,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:04:26,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:28,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:30,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:30,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:31,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:32,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:34,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:35,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:36,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:36,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:38,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:40,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:41,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:04:41,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:44,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:46,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:46,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:46,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:48,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:50,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:51,166 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:04:51,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:04:52,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:52,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:54,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:56,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:04:56,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:04:57,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:04:58,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:00,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:01,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:02,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:02,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:04,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:06,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:05:06,349 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:05:06,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:05:06,351 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:05:06,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:06,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:05:06,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:08,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:08,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:09,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:10,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:11,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:05:12,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:13,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:14,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:16,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:16,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:18,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:18,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:20,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:21,168 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:05:21,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:22,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:24,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:24,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:26,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:05:26,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:28,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:29,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:30,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:31,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:32,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:34,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:34,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:36,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:36,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:38,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:40,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:40,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:41,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:05:42,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:44,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:45,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:46,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:46,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:48,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:50,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:50,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:50,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:05:50,971 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:05:50,972 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:05:50,975 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:05:51,170 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:05:51,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:05:51,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:05:52,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:54,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:56,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:05:56,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:05:56,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:05:58,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:00,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:01,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:01,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:02,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:04,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:06,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:06,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:06,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:08,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:10,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:11,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:06:12,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:12,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:14,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:16,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:16,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:17,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:18,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:20,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:21,184 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:06:21,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:22,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:23,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:24,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:26,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:06:26,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:28,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:28,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:30,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:31,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:32,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:33,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:34,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:35,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:06:35,564 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:06:35,565 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:06:35,566 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:06:36,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:36,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:06:36,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:38,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:39,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:40,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:41,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:06:42,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:44,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:44,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:46,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:46,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:48,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:49,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:50,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:51,174 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:06:51,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:06:52,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:54,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:55,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:06:56,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:06:56,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:06:58,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:00,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:00,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:01,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:02,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:04,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:05,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:06,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:06,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:08,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:10,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:11,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:11,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:07:12,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:14,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:16,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:16,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:16,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:18,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:20,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:07:20,220 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:07:20,220 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:07:20,222 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:07:20,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:20,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:07:21,177 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:07:21,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:21,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:22,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:24,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:26,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:07:26,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:27,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:28,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:30,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:31,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:32,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:32,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:34,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:36,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:36,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:38,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:38,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:40,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:41,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:07:42,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:43,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:44,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:46,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:46,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:48,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:48,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:50,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:51,179 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:07:51,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:07:52,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:54,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:07:54,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:56,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:07:56,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:58,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:07:59,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:00,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:01,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:02,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:04,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:04,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:04,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:08:04,872 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:08:04,873 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:08:04,874 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:08:05,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:08:06,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:06,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:08,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:10,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:11,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:08:12,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:14,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:15,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:16,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:16,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:18,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:20,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:20,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:21,181 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:08:21,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:22,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:24,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:26,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:26,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:08:26,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:28,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:31,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:31,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:32,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:34,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:36,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:36,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:36,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:38,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:40,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:41,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:08:42,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:42,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:44,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:46,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:46,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:47,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:48,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:49,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:08:49,491 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:08:49,491 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:08:49,493 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:08:49,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:08:50,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:51,183 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:08:51,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:08:52,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:53,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:54,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:56,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:08:56,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:08:58,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:08:58,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:00,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:01,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:02,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:03,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:04,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:06,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:06,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:08,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:09,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:10,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:11,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:09:12,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:14,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:14,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:16,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:16,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:18,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:19,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:20,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:21,186 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:09:21,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:22,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:24,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:25,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:26,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:09:26,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:28,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:30,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:30,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:31,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:32,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:33,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:34,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:09:34,127 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:09:34,127 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:09:34,130 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:09:34,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:09:35,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:35,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:36,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:36,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:37,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:39,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:41,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:41,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:09:41,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:43,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:45,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:46,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:46,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:47,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:49,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:51,188 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:09:51,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:09:51,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:51,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:53,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:55,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:56,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:09:57,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:09:57,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:09:59,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:01,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:01,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:02,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:03,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:05,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:06,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:07,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:08,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:09,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:11,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:10:11,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:13,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:13,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:15,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:16,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:18,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:18,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:18,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:10:18,804 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:10:18,804 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:10:18,806 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:10:19,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:10:20,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:21,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:21,190 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:10:21,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:24,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:26,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:26,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:10:28,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:29,461 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:30,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:31,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:32,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:34,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:34,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:36,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:36,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:38,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:40,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:40,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:41,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:10:42,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:44,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:45,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:46,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:46,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:48,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:50,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:50,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:51,192 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:10:51,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:10:52,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:54,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:56,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:10:56,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:10:56,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:10:58,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:00,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:01,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:01,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:02,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:03,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:11:03,379 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:11:03,380 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:11:03,381 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:11:04,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:04,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:11:05,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:06,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:06,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:06,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:08,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:10,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:11,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:11:12,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:12,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:14,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:16,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:16,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:17,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:20,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:21,194 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:11:21,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:22,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:23,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:24,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:26,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:26,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:11:28,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:28,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:30,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:31,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:32,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:33,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:34,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:36,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:36,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:38,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:39,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:40,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:41,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:11:42,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:44,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:44,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:46,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:46,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:47,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:11:47,961 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:11:47,961 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:11:47,963 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:11:48,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:48,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:11:49,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:49,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:50,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:51,197 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:11:52,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:11:52,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:54,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:55,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:11:56,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:11:56,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:11:58,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:00,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:00,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:02,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:02,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:04,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:05,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:06,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:07,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:08,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:10,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:11,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:11,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:12:12,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:14,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:16,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:16,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:17,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:18,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:20,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:21,199 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:12:21,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:22,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:22,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:24,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:26,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:26,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:12:27,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:28,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:30,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:32,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:32,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:32,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:32,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:12:32,653 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:12:32,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:12:32,656 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:12:33,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:12:34,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:35,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:36,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:37,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:38,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:40,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:41,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:12:42,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:43,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:44,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:46,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:47,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:48,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:48,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:50,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:51,201 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:12:52,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:12:52,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:53,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:12:54,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:56,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:56,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:12:58,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:12:59,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:00,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:02,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:02,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:04,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:04,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:06,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:07,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:08,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:10,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:10,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:11,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:13:12,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:14,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:15,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:16,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:17,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:17,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:13:17,330 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:13:17,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:13:17,332 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:13:18,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:18,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:13:19,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:20,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:20,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:21,204 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:13:22,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:22,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:24,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:26,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:26,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:26,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:13:28,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:30,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:31,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:32,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:32,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:34,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:36,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:36,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:37,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:38,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:40,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:41,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:13:42,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:42,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:44,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:46,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:47,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:47,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:48,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:50,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:51,205 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:13:52,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:13:52,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:54,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:56,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:13:56,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:13:58,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:13:58,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:00,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:01,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:14:01,829 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:14:01,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:14:01,831 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:14:02,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:02,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:02,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:14:03,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:03,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:04,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:06,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:07,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:08,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:08,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:10,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:11,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:14:12,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:14,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:14,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:16,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:17,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:18,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:19,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:20,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:21,218 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:14:22,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:22,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:24,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:24,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:26,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:26,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:14:28,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:30,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:30,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:32,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:32,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:34,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:35,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:36,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:37,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:38,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:40,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:41,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:41,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:14:42,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:44,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:46,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:46,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:46,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:14:46,460 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:14:46,461 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:14:46,462 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:14:47,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:47,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:14:48,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:49,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:50,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:51,209 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:14:51,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:52,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:14:52,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:54,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:56,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:14:56,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:14:57,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:14:58,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:00,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:02,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:02,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:02,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:04,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:06,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:07,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:08,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:10,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:11,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:15:12,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:13,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:14,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:16,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:17,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:18,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:18,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:20,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:21,211 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:15:22,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:22,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:23,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:24,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:26,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:15:26,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:28,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:29,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:30,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:31,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:15:31,083 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:15:31,083 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:15:31,085 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:15:31,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:15:32,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:32,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:34,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:34,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:36,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:37,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:38,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:40,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:40,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:41,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:15:42,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:44,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:45,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:46,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:47,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:48,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:50,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:50,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:51,214 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:15:52,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:15:52,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:54,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:56,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:15:56,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:15:56,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:15:58,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:00,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:01,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:02,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:02,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:04,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:06,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:06,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:07,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:08,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:10,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:11,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:16:12,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:12,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:14,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:15,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:16:15,729 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:16:15,730 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:16:15,731 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:16:16,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:16,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:16:17,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:17,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:18,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:20,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:21,216 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:16:22,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:22,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:22,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:24,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:26,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:16:26,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:28,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:28,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:30,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:32,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:32,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:33,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:34,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:36,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:37,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:38,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:38,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:40,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:41,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:16:42,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:44,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:44,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:46,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:47,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:48,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:49,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:50,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:51,219 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:16:52,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:16:52,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:54,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:55,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:16:56,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:16:56,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:16:58,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:00,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:00,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:17:00,414 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:17:00,414 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:17:00,416 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:17:00,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:00,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:17:02,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:02,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:04,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:05,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:06,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:07,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:08,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:10,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:11,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:11,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:17:12,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:14,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:16,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:16,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:17,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:18,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:20,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:21,220 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:17:21,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:22,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:22,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:24,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:26,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:17:26,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:27,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:28,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:30,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:32,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:32,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:32,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:34,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:36,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:37,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:37,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:38,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:40,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:41,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:17:42,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:43,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:44,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:45,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:17:45,018 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:17:45,019 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:17:45,021 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:17:45,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:17:46,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:47,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:48,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:48,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:50,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:51,223 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:17:52,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:17:52,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:53,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:17:54,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:56,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:17:56,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:58,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:17:59,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:00,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:02,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:02,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:04,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:04,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:06,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:07,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:08,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:09,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:10,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:11,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:18:12,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:14,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:15,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:16,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:17,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:18,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:20,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:20,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:21,225 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:18:22,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:22,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:24,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:26,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:26,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:18:26,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:28,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:29,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:18:29,647 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:18:29,648 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:18:29,650 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:18:30,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:30,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:18:31,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:32,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:32,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:34,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:36,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:36,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:37,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:38,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:40,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:41,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:18:42,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:42,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:44,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:46,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:47,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:47,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:48,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:50,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:51,227 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:18:52,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:18:52,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:52,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:54,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:18:56,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:18:58,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:18:58,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:00,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:02,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:02,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:03,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:04,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:06,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:07,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:08,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:08,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:10,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:11,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:19:12,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:14,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:14,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:19:14,260 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:19:14,260 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:19:14,263 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:19:14,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:14,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:19:16,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:16,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:18,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:19,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:20,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:21,229 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:19:21,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:22,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:24,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:24,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:26,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:19:26,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:28,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:30,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:30,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:31,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:32,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:34,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:35,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:36,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:36,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:38,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:40,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:41,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:41,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:19:42,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:44,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:46,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:46,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:46,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:48,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:50,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:51,231 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:19:51,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:19:51,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:52,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:54,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:19:56,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:57,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:19:58,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:19:58,857 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:19:58,857 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:19:58,860 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:19:58,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:19:58,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:20:00,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:01,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:02,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:02,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:04,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:06,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:06,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:07,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:08,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:10,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:11,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:20:12,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:13,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:14,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:16,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:16,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:18,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:18,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:20,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:21,233 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:20:21,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:22,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:23,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:24,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:26,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:20:26,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:28,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:29,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:30,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:31,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:32,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:34,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:34,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:36,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:36,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:38,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:39,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:40,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:41,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:20:42,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:43,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:20:43,543 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:20:43,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:20:43,546 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:20:43,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:20:44,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:45,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:46,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:46,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:48,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:50,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:50,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:51,236 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:20:51,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:20:52,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:54,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:56,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:20:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:20:56,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:20:58,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:00,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:01,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:01,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:02,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:04,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:06,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:06,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:06,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:08,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:10,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:11,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:21:12,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:12,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:14,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:16,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:16,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:17,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:18,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:20,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:21,238 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:21:21,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:22,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:22,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:24,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:26,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:21:26,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:28,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:28,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:21:28,168 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:21:28,168 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:21:28,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:21:29,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:29,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:21:31,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:31,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:33,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:33,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:35,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:36,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:37,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:38,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:39,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:41,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:41,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:21:43,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:44,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:45,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:46,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:47,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:49,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:49,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:51,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:51,240 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:21:51,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:21:53,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:54,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:21:55,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:56,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:21:57,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:21:59,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:00,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:01,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:01,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:03,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:05,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:05,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:06,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:07,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:09,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:10,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:11,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:11,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:22:12,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:22:12,824 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:22:12,824 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:22:12,826 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:22:13,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:13,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:22:15,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:16,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:16,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:17,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:19,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:21,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:21,242 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:22:21,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:21,674 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:23,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:25,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:26,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:22:27,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:27,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:29,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:31,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:31,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:32,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:33,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:35,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:36,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:37,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:37,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:39,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:41,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:41,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:22:43,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:43,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:45,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:46,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:47,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:48,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:49,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:51,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:51,244 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:22:51,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:22:53,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:53,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:22:55,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:56,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:22:57,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:57,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:22:57,439 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:22:57,439 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:22:57,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:22:58,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:22:59,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:22:59,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:01,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:01,523 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:03,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:04,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:05,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:06,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:09,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:11,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:23:13,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:15,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:15,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:16,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:17,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:19,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:20,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:21,246 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:23:21,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:23,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:25,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:25,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:26,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:23:27,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:29,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:31,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:31,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:31,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:33,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:35,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:36,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:36,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:37,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:39,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:41,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:41,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:23:42,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:42,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:23:42,139 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:23:42,139 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:23:42,141 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:23:43,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:43,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:23:45,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:46,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:47,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:47,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:49,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:51,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:51,248 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:23:51,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:23:52,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:53,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:55,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:56,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:23:57,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:23:58,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:23:59,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:01,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:01,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:03,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:03,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:05,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:06,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:07,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:08,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:09,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:11,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:11,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:24:12,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:14,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:14,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:16,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:16,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:18,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:19,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:20,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:21,251 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:24:21,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:22,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:24,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:24,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:26,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:26,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:24:26,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:24:26,659 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:24:26,659 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:24:26,661 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:24:27,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:24:28,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:30,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:30,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:31,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:32,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:34,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:35,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:36,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:36,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:38,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:40,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:40,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:41,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:24:42,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:44,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:46,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:46,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:46,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:48,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:50,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:51,253 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:24:51,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:24:51,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:52,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:54,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:56,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:24:56,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:24:57,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:24:58,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:00,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:01,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:02,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:02,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:04,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:06,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:06,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:07,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:08,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:10,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:11,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:25:11,257 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:25:11,257 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:25:11,259 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:25:11,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:25:12,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:12,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:25:13,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:14,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:16,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:16,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:18,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:18,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:20,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:21,255 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:25:21,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:22,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:23,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:24,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:26,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:26,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:25:28,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:29,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:30,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:31,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:32,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:34,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:34,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:36,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:36,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:38,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:39,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:40,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:41,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:25:42,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:44,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:45,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:46,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:46,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:48,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:50,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:50,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:51,256 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:25:51,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:25:52,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:54,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:55,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:25:55,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:25:55,991 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:25:55,991 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:25:55,994 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:25:56,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:25:56,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:25:56,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:25:58,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:00,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:01,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:01,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:02,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:04,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:06,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:06,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:06,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:08,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:10,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:11,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:26:11,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:12,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:14,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:16,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:16,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:17,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:18,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:20,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:21,259 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:26:21,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:22,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:22,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:24,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:26,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:26:26,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:27,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:28,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:30,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:31,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:32,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:33,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:34,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:36,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:36,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:38,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:38,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:40,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:40,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:26:40,486 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:26:40,486 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:26:40,488 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:26:41,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:26:41,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:26:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:44,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:44,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:46,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:46,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:48,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:49,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:50,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:51,261 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:26:51,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:26:52,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:54,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:54,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:26:56,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:26:56,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:26:58,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:00,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:00,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:01,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:02,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:04,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:05,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:06,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:06,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:08,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:10,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:10,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:11,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:27:12,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:14,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:16,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:16,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:16,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:18,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:20,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:21,263 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:27:21,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:21,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:22,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:24,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:25,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:27:25,015 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:27:25,015 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:27:25,017 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:27:25,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:27:26,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:27:26,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:26,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:28,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:30,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:31,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:32,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:32,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:34,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:36,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:36,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:37,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:38,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:40,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:27:42,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:42,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:44,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:46,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:46,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:48,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:48,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:50,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:51,265 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:27:51,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:27:52,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:53,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:27:54,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:56,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:27:56,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:58,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:27:58,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:00,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:01,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:02,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:04,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:04,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:06,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:06,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:08,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:09,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:09,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:28:09,672 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:28:09,672 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:28:09,674 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:28:10,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:10,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:28:11,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:28:12,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:14,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:14,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:16,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:16,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:18,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:20,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:20,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:21,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:28:21,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:22,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:24,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:25,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:26,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:28:26,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:28,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:30,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:30,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:31,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:32,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:34,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:36,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:36,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:36,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:38,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:40,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:28:41,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:42,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:44,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:46,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:46,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:46,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:48,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:50,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:51,270 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:28:51,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:28:52,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:52,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:54,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:28:54,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:28:54,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:28:54,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:28:54,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:54,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:28:56,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:28:56,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:28:57,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:28:58,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:00,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:01,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:02,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:03,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:04,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:06,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:06,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:08,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:08,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:10,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:11,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:29:12,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:13,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:14,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:16,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:16,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:18,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:19,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:20,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:21,272 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:29:21,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:22,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:24,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:24,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:26,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:29:26,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:28,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:29,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:30,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:31,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:32,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:34,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:35,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:36,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:36,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:38,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:38,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:29:38,796 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:29:38,797 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:29:38,798 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:29:39,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:29:40,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:40,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:29:42,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:44,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:45,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:46,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:46,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:48,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:50,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:51,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:51,274 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:29:51,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:29:52,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:54,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:56,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:29:56,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:29:56,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:29:58,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:00,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:01,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:01,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:02,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:04,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:06,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:06,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:07,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:08,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:10,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:11,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:30:12,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:12,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:14,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:16,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:16,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:17,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:18,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:20,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:21,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:30:21,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:22,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:23,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:23,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:30:23,403 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:30:23,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:30:23,406 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:30:23,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:30:24,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:26,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:30:26,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:28,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:28,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:30,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:31,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:32,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:34,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:34,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:36,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:36,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:38,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:39,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:40,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:41,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:30:42,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:44,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:44,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:46,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:46,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:48,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:50,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:50,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:51,279 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:30:51,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:30:52,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:54,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:55,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:30:56,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:30:56,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:30:58,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:00,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:00,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:01,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:02,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:04,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:06,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:06,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:06,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:07,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:31:07,936 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:31:07,936 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:31:07,938 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:31:08,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:08,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:31:10,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:11,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:31:11,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:12,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:14,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:16,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:16,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:16,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:18,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:20,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:21,281 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:31:21,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:22,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:22,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:24,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:26,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:31:26,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:27,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:28,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:30,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:31,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:32,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:32,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:34,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:36,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:36,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:38,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:38,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:40,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:41,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:31:42,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:43,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:44,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:46,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:46,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:48,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:49,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:50,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:51,283 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:31:51,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:31:52,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:31:52,610 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:31:52,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:31:52,612 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:31:52,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:52,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:31:54,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:31:54,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:31:56,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:58,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:31:59,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:00,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:01,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:02,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:04,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:05,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:06,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:06,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:08,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:10,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:11,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:32:12,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:14,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:15,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:16,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:16,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:18,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:20,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:21,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:21,285 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:32:21,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:22,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:24,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:32:26,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:26,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:28,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:31,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:31,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:31,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:33,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:35,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:36,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:37,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:37,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:37,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:32:37,260 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:32:37,261 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:32:37,262 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:32:38,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:32:39,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:41,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:41,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:32:42,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:43,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:45,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:46,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:47,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:47,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:49,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:51,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:51,288 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:32:51,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:32:53,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:53,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:55,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:32:57,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:32:58,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:32:59,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:01,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:01,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:03,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:03,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:05,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:06,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:07,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:09,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:09,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:11,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:11,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:33:13,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:14,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:15,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:16,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:17,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:19,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:20,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:21,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:21,290 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:33:21,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:33:21,880 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:33:21,880 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:33:21,882 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:33:21,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:22,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:33:23,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:25,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:25,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:33:27,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:29,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:30,770 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:31,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:31,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:33,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:35,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:36,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:36,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:37,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:39,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:41,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:41,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:33:41,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:43,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:45,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:46,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:46,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:47,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:49,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:51,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:51,292 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:33:51,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:33:52,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:53,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:55,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:33:57,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:33:57,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:33:59,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:01,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:01,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:02,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:03,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:05,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:06,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:34:06,562 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:34:06,562 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:34:06,564 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:34:06,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:07,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:34:08,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:09,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:11,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:11,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:34:13,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:13,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:15,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:16,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:17,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:19,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:19,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:21,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:21,294 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:34:21,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:23,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:24,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:25,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:26,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:34:27,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:29,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:29,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:31,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:31,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:33,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:35,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:35,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:36,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:37,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:39,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:40,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:41,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:41,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:34:43,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:45,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:45,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:46,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:47,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:49,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:51,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:51,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:34:51,067 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:34:51,067 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:34:51,069 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:34:51,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:51,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:34:51,339 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:34:51,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:34:53,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:55,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:56,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:34:56,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:34:57,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:34:59,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:01,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:01,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:01,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:03,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:05,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:06,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:07,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:07,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:09,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:11,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:11,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:35:12,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:13,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:15,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:17,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:17,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:17,865 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:19,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:21,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:21,341 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:35:22,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:23,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:23,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:25,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:26,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:35:27,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:28,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:29,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:31,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:32,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:33,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:33,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:35,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:35,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:35:35,696 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:35:35,697 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:35:35,699 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:35:36,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:35:37,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:37,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:39,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:39,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:41,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:41,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:35:43,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:44,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:45,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:47,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:47,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:49,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:49,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:51,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:51,343 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:35:52,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:35:53,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:55,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:55,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:35:56,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:35:57,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:35:59,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:00,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:01,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:02,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:03,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:05,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:06,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:07,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:07,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:09,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:11,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:11,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:36:11,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:13,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:15,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:16,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:17,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:17,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:19,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:20,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:36:20,309 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:36:20,310 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:36:20,311 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:36:21,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:21,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:36:21,345 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:36:22,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:22,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:23,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:25,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:26,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:36:27,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:27,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:29,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:31,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:32,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:32,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:33,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:35,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:37,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:37,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:38,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:39,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:41,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:41,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:36:43,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:43,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:45,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:47,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:47,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:48,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:49,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:51,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:51,348 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:36:52,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:36:53,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:54,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:36:55,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:56,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:36:57,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:59,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:36:59,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:01,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:02,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:03,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:04,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:04,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:37:04,915 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:37:04,915 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:37:04,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:37:05,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:05,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:37:07,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:07,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:09,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:10,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:11,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:11,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:37:13,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:15,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:15,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:17,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:17,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:19,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:21,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:21,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:21,350 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:37:22,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:23,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:25,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:26,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:37:26,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:27,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:29,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:31,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:31,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:32,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:33,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:35,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:37,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:37,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:37,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:39,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:41,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:37:41,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:42,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:43,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:45,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:47,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:47,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:47,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:49,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:49,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:37:49,600 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:37:49,600 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:37:49,602 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:37:50,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:37:51,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:51,363 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:37:52,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:37:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:53,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:55,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:56,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:37:57,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:37:58,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:37:59,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:01,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:02,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:03,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:03,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:05,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:07,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:07,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:09,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:09,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:11,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:38:11,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:13,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:14,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:15,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:17,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:17,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:19,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:19,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:21,354 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:38:21,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:22,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:23,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:25,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:25,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:26,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:38:27,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:29,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:30,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:31,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:32,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:33,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:34,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:38:34,285 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:38:34,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:38:34,288 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:38:34,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:38:35,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:36,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:37,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:37,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:39,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:41,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:38:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:41,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:43,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:46,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:47,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:47,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:48,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:50,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:51,357 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:38:52,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:52,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:38:52,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:54,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:56,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:38:56,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:38:57,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:38:58,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:00,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:02,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:02,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:02,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:04,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:06,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:07,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:08,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:08,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:10,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:39:12,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:13,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:14,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:16,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:17,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:18,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:18,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:18,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:39:18,885 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:39:18,885 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:39:18,887 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:39:19,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:39:20,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:21,359 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:39:22,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:22,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:24,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:24,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:26,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:39:26,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:28,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:29,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:30,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:32,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:32,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:34,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:34,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:36,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:37,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:38,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:40,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:41,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:39:42,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:44,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:45,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:46,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:47,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:48,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:50,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:50,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:51,361 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:39:52,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:39:52,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:54,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:56,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:39:56,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:39:56,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:39:58,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:00,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:01,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:02,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:02,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:03,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:40:03,443 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:40:03,444 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:40:03,445 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:40:03,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:40:04,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:06,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:06,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:07,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:08,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:10,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:40:12,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:12,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:14,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:16,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:17,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:17,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:18,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:20,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:21,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:40:22,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:22,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:23,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:24,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:26,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:40:26,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:28,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:28,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:30,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:32,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:32,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:33,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:34,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:36,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:37,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:38,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:39,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:40,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:41,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:40:42,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:44,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:44,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:46,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:47,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:48,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:40:48,305 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:40:48,306 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:40:48,308 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:40:48,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:48,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:40:50,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:50,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:51,366 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:40:52,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:40:52,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:54,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:55,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:40:56,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:40:56,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:40:58,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:00,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:00,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:02,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:02,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:04,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:06,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:06,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:07,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:08,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:10,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:41:11,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:12,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:14,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:16,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:16,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:17,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:18,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:20,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:21,368 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:41:22,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:22,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:22,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:24,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:26,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:41:26,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:27,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:28,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:30,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:32,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:32,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:32,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:32,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:41:32,946 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:41:32,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:41:32,947 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:41:33,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:41:34,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:36,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:37,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:38,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:38,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:40,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:41,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:41:42,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:43,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:44,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:46,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:47,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:48,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:49,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:50,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:51,370 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:41:52,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:41:52,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:54,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:41:54,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:56,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:41:56,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:58,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:41:59,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:00,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:02,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:02,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:04,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:05,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:06,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:07,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:08,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:10,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:10,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:11,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:42:12,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:14,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:15,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:16,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:17,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:17,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:42:17,632 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:42:17,632 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:42:17,634 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:42:17,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:42:18,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:20,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:21,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:21,372 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:42:22,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:22,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:24,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:26,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:42:26,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:26,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:28,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:30,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:31,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:32,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:32,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:34,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:36,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:37,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:37,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:38,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:40,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:41,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:42:42,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:42,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:44,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:46,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:47,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:47,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:48,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:50,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:51,375 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:42:52,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:42:52,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:53,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:54,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:56,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:42:56,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:42:58,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:42:58,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:00,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:02,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:43:02,183 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:43:02,184 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:43:02,185 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:43:02,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:02,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:02,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:43:03,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:04,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:06,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:07,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:08,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:09,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:10,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:11,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:43:12,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:14,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:14,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:16,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:17,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:18,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:19,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:20,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:21,377 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:43:22,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:22,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:24,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:25,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:26,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:43:26,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:28,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:30,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:30,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:32,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:32,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:35,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:35,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:37,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:37,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:39,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:41,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:41,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:41,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:43:43,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:45,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:46,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:46,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:43:46,671 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:43:46,671 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:43:46,673 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:43:47,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:47,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:43:47,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:49,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:51,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:51,379 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:43:51,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:52,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:43:53,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:55,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:56,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:43:57,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:43:57,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:43:59,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:01,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:02,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:02,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:03,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:05,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:07,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:07,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:07,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:09,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:11,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:11,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:44:13,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:13,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:15,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:17,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:17,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:18,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:19,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:21,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:21,381 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:44:22,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:23,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:24,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:25,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:26,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:44:27,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:29,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:29,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:31,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:31,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:44:31,251 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:44:31,252 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:44:31,253 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:44:32,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:44:32,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:33,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:34,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:35,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:37,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:37,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:39,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:40,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:41,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:41,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:44:43,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:45,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:45,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:47,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:47,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:49,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:50,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:51,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:51,384 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:44:52,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:44:53,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:55,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:56,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:44:56,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:44:57,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:44:59,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:01,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:01,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:01,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:03,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:05,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:06,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:06,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:07,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:09,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:11,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:11,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:45:12,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:13,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:15,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:15,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:45:15,885 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:45:15,885 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:45:15,888 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:45:16,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:45:16,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:17,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:17,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:19,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:21,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:21,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:21,385 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:45:22,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:23,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:25,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:26,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:45:27,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:28,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:29,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:31,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:31,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:33,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:33,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:35,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:36,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:37,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:39,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:39,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:41,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:41,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:45:43,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:44,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:45,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:46,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:47,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:49,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:49,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:51,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:51,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:45:51,387 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:45:53,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:45:55,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:56,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:45:57,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:45:59,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:00,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:00,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:46:00,501 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:46:00,501 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:46:00,504 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:46:01,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:01,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:46:01,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:03,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:05,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:05,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:06,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:07,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:09,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:11,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:11,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:11,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:46:13,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:15,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:16,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:16,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:17,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:19,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:21,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:21,390 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:46:21,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:21,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:23,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:25,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:26,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:46:27,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:27,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:29,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:31,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:31,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:32,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:33,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:35,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:36,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:37,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:37,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:39,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:41,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:41,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:46:43,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:43,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:45,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:46:45,139 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:46:45,140 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:46:45,141 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:46:45,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:45,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:46:46,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:47,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:48,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:49,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:51,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:51,392 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:46:51,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:46:53,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:54,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:46:55,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:56,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:46:57,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:59,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:46:59,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:01,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:01,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:03,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:04,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:05,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:06,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:07,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:09,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:10,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:11,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:11,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:47:13,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:15,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:15,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:16,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:17,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:19,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:20,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:21,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:21,395 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:47:21,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:23,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:25,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:26,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:26,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:47:27,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:29,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:29,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:47:29,736 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:47:29,736 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:47:29,739 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:47:30,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:47:31,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:31,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:31,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:33,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:35,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:36,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:36,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:37,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:39,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:41,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:41,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:47:42,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:43,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:45,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:46,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:47,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:47,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:49,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:51,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:51,397 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:47:51,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:47:52,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:53,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:55,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:56,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:47:57,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:47:58,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:47:59,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:01,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:01,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:03,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:03,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:05,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:06,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:07,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:09,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:09,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:11,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:11,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:48:13,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:14,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:14,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:48:14,397 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:48:14,397 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:48:14,399 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:48:15,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:15,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:48:16,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:17,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:19,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:19,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:21,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:21,399 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:48:21,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:23,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:25,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:25,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:26,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:48:27,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:29,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:30,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:31,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:31,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:33,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:35,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:35,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:36,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:37,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:39,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:41,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:41,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:48:41,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:43,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:45,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:46,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:46,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:47,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:49,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:51,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:51,402 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:48:51,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:48:51,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:53,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:55,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:56,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:48:57,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:48:57,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:59,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:48:59,003 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:48:59,003 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:48:59,005 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:48:59,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:48:59,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:49:01,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:01,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:02,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:03,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:05,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:06,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:07,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:07,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:09,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:11,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:49:11,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:13,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:13,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:15,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:16,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:17,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:18,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:19,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:21,404 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:49:21,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:21,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:23,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:23,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:25,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:26,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:49:27,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:29,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:29,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:31,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:31,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:33,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:34,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:35,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:36,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:37,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:39,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:40,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:41,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:49:41,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:43,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:43,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:49:43,655 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:49:43,656 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:49:43,657 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:49:44,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:49:45,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:45,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:46,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:47,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:49,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:50,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:51,416 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:49:51,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:49:51,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:53,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:55,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:56,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:49:56,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:49:57,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:49:59,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:01,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:01,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:01,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:03,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:05,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:06,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:06,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:07,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:09,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:11,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:50:11,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:12,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:13,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:14,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:16,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:16,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:17,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:18,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:20,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:21,408 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:50:21,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:22,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:22,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:24,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:26,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:50:26,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:28,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:28,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:50:28,265 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:50:28,265 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:50:28,267 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:50:28,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:28,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:50:30,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:31,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:32,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:33,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:34,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:36,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:36,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:38,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:38,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:40,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:41,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:50:42,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:44,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:44,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:46,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:46,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:48,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:49,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:50,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:51,410 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:50:51,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:50:52,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:54,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:54,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:50:56,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:50:56,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:50:58,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:00,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:00,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:01,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:02,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:04,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:05,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:06,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:06,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:08,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:10,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:11,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:11,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:51:12,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:12,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:51:12,863 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:51:12,863 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:51:12,866 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:51:13,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:51:14,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:16,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:16,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:16,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:18,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:20,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:21,413 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:51:21,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:21,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:22,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:24,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:26,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:51:26,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:27,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:28,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:30,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:31,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:32,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:32,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:34,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:36,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:36,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:37,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:38,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:40,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:41,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:51:42,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:43,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:44,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:46,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:46,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:48,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:48,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:50,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:51,414 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:51:51,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:51:52,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:53,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:51:54,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:56,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:51:56,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:57,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:51:57,399 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:51:57,399 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:51:57,401 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:51:57,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:51:58,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:51:59,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:00,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:01,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:02,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:04,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:04,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:06,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:06,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:08,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:09,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:10,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:11,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:52:12,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:14,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:15,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:16,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:16,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:18,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:20,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:20,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:21,416 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:52:21,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:22,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:24,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:25,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:26,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:52:26,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:28,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:30,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:31,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:31,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:32,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:34,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:36,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:36,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:36,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:38,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:40,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:41,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:52:41,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:41,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:52:41,891 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:52:41,891 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:52:41,893 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:52:42,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:42,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:52:44,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:46,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:46,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:47,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:48,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:50,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:51,419 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:52:51,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:52:52,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:52,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:54,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:56,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:52:56,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:52:57,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:52:58,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:00,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:01,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:02,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:03,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:04,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:06,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:06,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:08,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:08,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:10,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:11,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:53:12,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:14,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:14,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:16,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:16,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:18,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:19,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:20,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:21,421 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:53:21,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:22,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:24,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:24,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:26,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:53:26,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:53:26,551 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:53:26,551 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:53:26,553 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:53:26,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:26,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:53:28,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:30,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:30,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:31,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:32,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:34,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:35,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:36,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:36,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:38,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:40,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:40,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:41,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:53:42,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:44,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:46,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:46,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:46,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:48,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:50,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:51,423 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:53:51,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:51,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:53:52,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:54,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:56,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:53:56,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:53:56,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:53:58,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:00,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:01,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:02,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:02,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:04,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:06,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:06,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:07,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:08,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:10,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:11,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:54:11,069 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:54:11,069 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:54:11,071 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:54:11,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:54:11,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:54:12,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:12,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:14,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:16,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:16,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:18,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:18,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:20,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:21,425 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:54:21,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:22,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:23,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:24,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:26,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:54:26,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:28,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:28,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:30,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:31,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:32,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:34,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:34,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:36,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:36,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:38,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:39,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:40,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:41,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:54:42,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:44,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:44,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:46,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:46,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:48,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:50,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:50,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:51,428 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:54:51,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:54:52,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:54,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:55,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:54:55,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:54:55,520 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:54:55,521 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:54:55,522 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:54:55,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:54:56,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:54:56,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:54:58,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:00,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:00,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:01,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:02,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:04,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:06,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:06,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:06,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:08,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:10,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:11,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:55:11,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:12,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:14,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:16,701 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:16,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:16,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:18,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:20,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:21,430 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:55:21,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:22,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:22,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:24,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:26,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:55:26,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:27,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:28,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:30,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:31,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:32,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:32,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:34,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:36,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:36,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:38,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:38,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:40,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:55:40,147 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:55:40,147 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:55:40,149 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:55:40,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:40,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:55:41,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:55:42,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:43,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:45,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:46,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:47,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:49,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:49,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:51,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:51,432 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:55:51,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:55:53,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:54,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:55:55,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:56,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:55:57,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:59,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:55:59,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:01,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:01,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:03,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:05,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:05,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:06,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:07,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:09,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:10,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:11,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:11,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:56:13,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:15,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:15,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:16,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:17,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:19,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:21,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:21,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:21,434 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:56:21,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:23,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:24,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:56:24,710 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:56:24,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:56:24,712 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:56:25,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:25,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:56:26,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:56:26,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:27,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:29,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:31,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:31,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:31,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:33,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:35,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:36,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:37,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:37,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:39,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:41,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:41,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:56:42,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:43,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:45,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:46,766 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:47,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:47,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:49,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:51,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:51,437 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:56:51,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:56:53,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:53,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:55,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:56,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:56:57,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:56:58,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:56:59,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:01,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:01,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:03,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:03,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:05,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:06,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:07,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:09,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:09,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:09,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:57:09,194 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:57:09,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:57:09,196 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:57:10,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:57:11,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:11,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:57:13,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:14,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:15,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:16,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:17,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:19,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:19,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:21,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:21,439 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:57:21,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:23,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:25,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:25,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:26,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:57:27,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:29,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:30,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:31,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:31,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:33,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:35,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:35,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:36,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:37,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:39,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:41,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:41,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:41,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:57:43,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:45,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:46,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:46,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:47,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:49,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:51,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:51,441 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:57:51,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:57:52,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:53,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:53,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:57:53,827 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:57:53,828 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:57:53,830 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:57:54,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:57:55,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:56,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:57:57,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:57:57,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:57:59,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:01,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:01,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:02,694 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:03,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:05,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:06,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:07,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:08,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:09,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:11,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:11,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:58:13,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:13,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:15,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:16,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:17,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:18,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:19,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:21,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:21,443 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:58:21,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:23,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:24,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:25,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:26,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:58:27,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:29,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:29,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:31,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:31,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:33,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:34,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:35,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:36,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:37,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:38,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:58:38,373 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:58:38,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:58:38,375 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:58:39,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:39,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:58:40,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:41,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:41,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:58:43,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:45,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:45,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:46,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:47,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:49,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:50,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:51,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:51,447 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:58:51,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:58:53,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:55,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:56,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:58:56,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:58:57,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:58:59,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:01,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:01,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:01,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:03,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:05,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:06,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:06,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:07,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:09,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:11,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:11,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:59:12,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:13,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:15,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:16,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:17,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:17,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:19,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:21,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:21,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:59:21,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:22,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:22,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 22:59:22,942 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 22:59:22,942 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 22:59:22,944 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 22:59:23,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:23,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 22:59:25,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:26,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:59:27,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:28,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:29,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:31,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:31,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:33,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:33,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:35,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:37,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:38,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:39,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:41,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:59:41,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:43,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:44,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:45,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:46,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:47,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:49,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:49,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:51,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:51,452 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 22:59:51,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 22:59:53,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:55,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 22:59:55,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:56,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 22:59:57,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 22:59:59,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:00,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:01,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:01,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:03,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:05,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:05,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:06,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:07,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:07,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:00:07,506 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:00:07,506 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:00:07,508 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:00:08,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:00:09,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:11,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:11,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:00:11,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:13,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:15,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:16,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:16,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:17,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:19,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:21,454 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:00:21,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:21,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:21,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:23,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:25,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:26,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:00:27,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:27,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:29,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:31,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:31,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:32,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:33,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:35,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:36,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:37,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:37,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:39,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:00:41,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:43,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:43,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:45,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:46,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:47,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:48,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:49,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:51,457 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:00:51,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:51,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:00:52,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:00:52,002 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:00:52,003 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:00:52,005 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:00:52,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:00:53,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:53,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:55,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:56,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:00:57,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:00:59,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:00:59,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:01,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:01,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:03,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:04,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:05,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:06,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:07,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:09,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:09,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:11,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:01:11,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:13,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:15,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:15,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:16,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:17,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:19,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:20,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:21,459 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:01:21,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:21,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:23,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:25,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:25,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:26,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:01:27,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:29,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:31,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:31,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:31,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:33,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:35,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:36,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:36,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:01:36,579 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:01:36,579 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:01:36,582 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:01:36,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:01:36,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:37,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:39,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:01:41,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:41,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:43,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:45,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:46,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:47,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:47,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:49,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:51,474 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:01:51,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:51,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:01:52,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:53,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:55,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:56,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:01:57,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:01:57,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:01:59,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:01,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:01,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:03,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:03,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:05,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:06,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:07,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:08,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:09,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:11,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:02:11,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:13,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:13,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:15,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:16,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:17,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:19,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:19,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:21,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:02:21,077 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:02:21,077 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:02:21,078 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:02:21,464 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:02:21,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:21,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:02:21,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:23,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:24,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:25,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:26,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:02:27,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:29,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:30,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:31,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:31,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:33,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:35,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:35,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:36,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:37,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:39,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:40,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:41,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:02:41,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:43,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:45,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:46,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:46,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:47,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:49,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:51,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:51,466 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:02:51,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:51,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:02:53,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:55,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:56,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:02:56,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:02:57,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:02:59,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:01,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:01,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:02,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:03,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:05,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:03:05,603 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:03:05,603 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:03:05,605 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:03:05,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:05,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:03:06,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:07,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:07,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:09,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:11,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:03:11,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:12,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:13,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:15,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:17,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:17,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:18,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:19,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:21,467 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:03:21,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:22,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:23,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:23,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:25,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:26,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:03:27,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:28,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:29,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:31,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:32,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:33,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:34,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:35,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:37,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:39,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:41,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:03:41,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:43,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:44,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:45,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:47,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:47,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:49,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:50,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:50,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:03:50,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:03:50,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:03:50,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:03:50,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:03:51,470 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:03:51,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:52,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:03:53,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:55,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:03:55,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:56,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:03:57,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:03:59,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:00,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:01,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:02,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:03,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:05,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:06,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:07,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:07,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:09,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:11,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:04:11,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:11,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:13,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:15,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:16,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:17,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:17,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:19,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:21,472 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:04:21,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:22,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:22,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:23,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:25,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:26,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:04:27,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:27,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:29,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:31,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:32,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:32,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:33,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:34,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:04:34,651 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:04:34,651 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:04:34,653 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:04:34,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:04:35,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:37,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:37,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:38,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:39,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:41,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:04:41,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:43,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:43,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:45,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:47,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:47,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:48,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:49,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:51,474 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:04:51,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:52,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:04:53,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:54,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:55,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:56,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:04:57,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:04:59,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:04:59,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:01,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:02,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:03,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:04,948 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:05,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:07,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:07,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:09,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:10,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:11,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:05:11,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:13,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:15,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:15,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:17,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:17,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:19,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:05:19,296 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:05:19,297 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:05:19,299 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:05:19,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:19,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:05:21,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:21,477 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:05:21,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:22,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:23,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:25,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:26,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:05:26,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:27,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:29,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:31,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:31,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:32,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:33,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:35,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:37,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:37,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:37,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:39,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:41,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:05:41,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:42,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:43,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:45,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:47,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:47,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:47,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:49,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:51,479 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:05:51,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:52,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:05:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:53,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:55,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:56,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:05:57,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:05:58,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:05:59,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:01,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:02,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:03,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:03,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:06:03,861 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:06:03,862 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:06:03,863 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:06:03,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:03,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:06:05,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:07,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:07,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:09,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:09,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:11,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:06:11,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:13,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:14,523 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:15,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:17,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:17,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:19,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:19,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:21,481 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:06:21,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:22,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:23,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:25,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:25,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:26,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:06:27,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:29,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:30,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:31,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:32,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:33,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:35,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:35,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:37,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:37,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:39,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:41,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:41,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:06:41,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:43,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:45,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:46,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:47,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:47,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:48,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:06:48,351 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:06:48,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:06:48,353 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:06:48,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:06:49,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:51,484 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:06:51,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:51,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:52,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:06:53,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:55,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:56,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:06:57,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:06:57,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:06:59,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:01,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:02,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:02,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:02,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:04,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:06,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:07,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:08,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:10,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:11,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:07:12,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:13,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:14,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:16,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:17,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:18,732 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:18,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:20,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:21,486 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:07:22,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:22,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:24,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:24,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:26,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:07:26,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:28,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:29,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:30,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:32,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:32,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:33,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:07:33,041 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:07:33,041 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:07:33,043 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:07:33,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:07:34,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:34,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:36,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:37,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:38,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:40,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:40,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:41,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:07:42,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:44,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:45,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:46,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:47,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:48,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:50,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:50,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:51,498 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:07:52,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:07:52,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:54,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:56,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:07:56,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:07:56,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:07:58,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:01,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:01,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:02,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:03,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:05,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:06,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:07,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:07,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:09,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:11,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:11,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:08:12,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:13,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:15,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:17,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:17,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:17,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:17,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:08:17,547 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:08:17,547 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:08:17,549 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:08:18,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:08:19,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:21,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:21,490 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:08:22,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:22,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:23,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:25,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:26,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:08:27,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:28,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:29,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:31,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:32,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:33,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:33,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:35,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:37,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:37,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:39,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:39,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:41,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:41,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:08:43,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:44,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:45,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:47,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:47,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:49,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:49,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:51,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:51,492 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:08:52,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:08:53,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:55,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:55,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:08:56,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:08:57,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:08:59,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:00,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:01,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:02,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:02,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:09:02,239 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:09:02,239 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:09:02,241 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:09:03,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:03,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:09:05,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:05,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:07,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:07,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:09,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:11,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:11,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:11,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:09:13,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:15,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:16,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:17,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:17,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:19,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:21,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:21,494 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:09:21,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:22,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:23,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:25,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:26,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:09:27,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:27,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:29,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:31,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:32,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:32,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:33,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:35,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:37,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:37,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:37,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:39,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:41,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:41,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:09:43,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:43,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:45,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:46,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:09:46,875 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:09:46,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:09:46,877 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:09:47,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:09:48,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:49,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:51,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:51,507 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:09:52,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:09:53,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:53,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:09:55,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:56,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:09:57,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:59,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:09:59,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:01,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:02,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:03,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:04,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:05,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:07,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:07,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:09,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:09,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:11,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:11,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:10:13,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:15,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:15,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:17,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:17,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:19,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:20,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:21,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:21,499 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:10:22,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:23,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:25,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:26,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:26,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:10:27,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:29,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:31,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:31,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:31,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:10:31,446 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:10:31,446 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:10:31,447 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:10:32,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:32,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:10:33,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:35,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:36,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:37,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:37,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:39,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:41,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:10:41,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:42,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:43,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:45,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:47,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:47,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:47,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:49,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:51,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:51,501 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:10:52,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:10:52,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:53,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:55,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:56,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:10:57,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:10:58,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:10:59,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:01,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:02,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:03,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:03,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:05,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:07,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:07,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:08,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:09,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:11,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:11:11,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:13,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:14,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:15,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:16,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:11:16,089 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:11:16,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:11:16,092 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:11:16,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:11:17,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:17,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:19,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:19,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:21,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:21,504 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:11:22,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:23,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:24,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:25,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:26,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:11:27,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:29,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:30,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:31,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:32,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:33,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:35,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:35,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:37,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:37,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:39,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:40,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:41,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:11:41,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:43,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:45,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:46,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:47,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:47,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:49,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:51,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:51,518 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:11:51,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:52,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:11:53,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:55,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:56,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:11:57,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:11:57,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:11:59,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:00,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:12:00,610 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:12:00,611 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:12:00,613 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:12:01,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:01,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:12:02,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:02,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:03,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:05,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:07,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:07,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:07,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:09,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:11,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:12:11,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:13,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:13,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:15,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:17,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:17,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:18,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:19,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:21,508 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:12:21,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:22,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:23,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:23,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:25,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:26,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:12:27,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:29,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:29,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:31,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:32,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:33,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:34,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:35,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:37,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:37,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:39,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:39,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:41,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:12:41,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:43,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:45,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:12:45,083 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:12:45,083 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:12:45,084 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:12:45,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:45,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:12:47,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:47,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:49,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:50,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:51,510 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:12:51,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:52,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:12:53,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:55,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:55,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:12:56,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:12:57,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:12:59,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:01,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:01,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:02,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:03,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:05,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:06,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:07,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:07,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:09,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:11,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:13:11,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:11,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:13,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:15,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:17,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:17,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:17,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:19,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:21,513 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:13:21,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:22,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:22,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:23,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:25,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:26,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:13:27,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:27,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:29,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:29,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:13:29,661 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:13:29,661 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:13:29,663 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:13:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:13:31,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:32,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:33,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:33,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:35,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:37,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:37,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:38,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:39,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:41,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:13:41,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:43,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:43,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:45,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:47,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:47,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:49,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:49,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:51,525 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:13:51,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:52,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:13:53,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:54,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:13:55,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:56,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:13:57,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:59,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:13:59,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:01,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:02,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:03,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:05,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:05,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:07,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:07,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:09,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:10,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:11,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:14:11,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:13,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:14,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:14:14,259 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:14:14,260 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:14:14,261 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:14:14,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:14:15,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:16,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:16,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:17,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:19,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:21,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:21,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:21,517 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:14:21,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:23,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:25,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:26,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:14:26,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:27,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:29,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:31,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:31,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:32,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:33,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:35,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:36,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:37,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:37,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:39,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:41,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:14:41,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:42,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:43,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:45,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:46,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:47,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:48,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:49,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:51,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:14:51,519 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:14:51,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:53,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:53,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:55,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:56,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:14:57,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:58,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:14:58,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:14:58,737 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:14:58,737 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:14:58,739 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:14:59,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:14:59,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:15:01,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:01,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:03,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:04,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:05,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:06,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:07,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:09,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:09,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:11,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:15:11,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:13,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:14,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:15,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:16,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:17,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:19,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:20,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:21,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:21,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:15:21,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:23,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:25,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:25,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:26,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:15:27,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:29,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:30,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:31,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:31,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:33,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:35,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:36,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:36,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:37,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:39,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:41,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:15:41,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:41,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:43,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:15:43,287 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:15:43,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:15:43,289 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:15:43,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:43,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:15:45,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:46,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:46,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:47,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:49,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:51,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:15:51,534 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:15:51,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:52,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:53,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:55,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:56,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:15:57,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:15:57,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:15:59,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:01,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:01,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:02,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:03,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:05,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:06,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:07,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:08,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:09,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:11,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:16:11,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:13,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:13,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:15,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:16,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:17,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:18,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:19,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:21,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:21,526 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:16:21,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:23,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:24,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:25,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:26,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:16:27,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:27,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:16:27,851 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:16:27,851 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:16:27,853 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:16:28,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:16:29,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:29,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:31,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:31,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:33,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:34,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:35,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:36,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:37,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:39,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:40,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:41,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:16:41,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:43,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:45,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:45,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:46,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:47,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:49,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:50,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:51,528 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:16:51,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:16:51,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:53,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:55,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:56,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:16:56,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:16:57,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:16:59,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:01,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:01,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:01,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:03,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:05,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:06,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:06,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:07,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:09,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:11,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:17:11,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:12,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:12,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:17:12,347 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:17:12,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:17:12,350 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:17:12,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:17:13,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:15,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:16,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:17,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:17,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:19,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:21,530 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:17:21,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:21,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:22,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:23,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:25,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:26,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:17:27,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:28,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:29,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:31,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:31,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:33,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:33,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:35,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:36,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:37,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:39,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:39,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:41,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:17:41,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:43,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:44,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:45,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:46,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:47,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:49,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:49,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:51,532 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:17:51,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:17:51,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:53,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:55,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:17:55,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:56,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:17:56,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:17:56,903 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:17:56,903 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:17:56,905 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:17:57,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:17:57,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:17:59,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:00,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:01,595 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:01,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:03,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:05,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:05,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:06,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:07,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:09,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:11,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:11,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:18:11,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:13,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:15,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:16,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:16,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:17,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:19,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:21,535 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:18:21,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:21,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:21,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:23,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:25,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:26,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:18:27,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:27,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:29,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:31,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:31,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:32,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:33,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:35,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:36,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:37,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:37,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:39,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:41,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:18:41,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:18:41,443 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:18:41,443 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:18:41,444 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:18:41,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:41,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:18:43,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:43,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:45,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:46,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:47,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:48,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:49,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:51,537 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:18:51,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:18:51,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:53,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:53,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:55,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:56,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:18:57,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:18:59,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:18:59,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:01,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:01,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:03,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:04,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:05,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:06,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:07,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:09,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:09,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:11,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:19:11,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:13,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:15,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:15,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:16,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:18,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:20,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:20,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:21,539 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:19:21,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:22,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:24,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:25,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:25,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:19:25,996 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:19:25,996 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:19:25,998 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:19:26,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:26,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:19:26,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:19:28,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:30,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:31,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:31,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:32,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:34,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:36,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:36,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:36,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:38,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:40,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:41,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:19:42,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:42,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:44,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:46,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:46,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:47,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:48,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:50,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:51,551 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:19:51,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:19:52,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:52,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:54,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:56,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:19:56,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:19:58,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:19:58,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:00,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:01,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:02,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:03,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:04,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:06,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:06,759 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:07,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:08,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:09,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:10,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:20:10,534 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:20:10,535 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:20:10,536 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:20:11,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:11,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:20:11,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:20:13,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:14,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:15,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:16,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:17,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:19,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:19,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:21,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:21,544 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:20:21,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:23,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:25,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:26,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:20:27,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:29,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:30,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:31,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:31,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:33,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:35,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:35,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:36,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:37,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:39,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:40,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:41,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:41,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:20:43,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:45,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:46,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:46,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:47,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:49,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:51,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:51,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:51,546 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:20:51,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:20:53,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:54,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:20:54,990 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:20:54,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:20:54,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:20:55,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:55,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:20:56,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:20:56,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:20:57,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:20:59,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:01,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:01,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:02,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:03,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:05,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:06,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:07,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:07,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:09,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:11,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:11,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:21:12,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:13,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:15,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:16,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:17,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:18,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:19,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:21,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:21,548 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:21:21,817 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:23,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:23,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:25,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:26,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:21:27,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:28,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:29,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:31,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:31,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:33,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:34,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:35,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:36,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:37,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:39,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:39,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:39,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:21:39,588 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:21:39,589 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:21:39,590 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:21:40,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:21:41,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:41,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:21:43,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:44,893 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:45,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:46,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:47,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:49,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:50,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:51,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:51,561 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:21:51,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:21:53,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:55,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:55,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:21:56,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:21:57,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:21:59,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:00,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:01,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:01,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:03,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:05,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:06,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:06,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:07,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:09,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:11,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:11,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:22:11,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:13,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:15,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:16,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:17,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:19,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:21,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:21,552 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:22:21,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:22,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:23,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:24,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:22:24,104 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:22:24,105 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:22:24,107 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:22:24,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:22:25,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:26,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:22:27,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:27,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:29,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:31,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:31,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:32,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:33,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:35,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:36,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:37,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:38,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:39,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:41,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:41,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:22:43,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:43,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:45,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:46,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:47,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:48,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:49,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:51,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:51,555 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:22:51,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:22:53,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:54,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:22:55,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:56,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:22:57,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:59,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:22:59,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:01,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:01,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:03,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:04,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:05,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:06,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:07,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:08,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:23:08,565 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:23:08,565 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:23:08,567 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:23:09,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:09,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:23:10,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:11,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:23:11,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:13,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:15,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:15,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:16,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:17,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:19,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:21,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:21,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:21,557 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:23:21,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:23,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:25,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:26,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:26,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:23:27,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:29,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:31,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:31,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:31,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:33,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:35,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:36,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:37,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:37,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:39,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:41,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:23:41,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:42,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:43,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:45,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:46,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:47,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:47,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:49,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:51,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:51,570 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:23:51,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:23:53,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:53,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:23:53,152 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:23:53,152 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:23:53,155 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:23:53,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:53,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:23:55,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:56,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:23:57,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:23:58,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:23:59,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:01,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:01,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:03,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:03,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:05,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:06,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:07,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:09,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:09,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:11,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:24:11,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:13,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:14,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:15,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:16,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:17,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:19,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:19,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:21,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:21,561 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:24:21,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:23,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:25,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:26,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:24:27,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:29,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:30,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:31,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:31,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:33,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:35,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:35,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:36,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:37,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:37,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:24:37,700 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:24:37,701 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:24:37,702 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:24:38,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:24:39,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:41,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:41,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:24:41,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:43,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:45,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:46,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:46,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:47,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:49,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:51,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:51,564 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:24:51,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:51,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:24:53,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:55,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:56,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:24:57,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:24:57,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:24:59,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:01,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:01,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:02,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:03,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:05,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:06,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:07,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:07,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:09,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:11,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:25:11,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:13,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:13,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:15,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:17,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:18,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:19,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:21,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:21,566 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:25:21,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:22,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:25:22,167 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:25:22,168 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:25:22,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:25:22,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:25:23,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:23,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:25,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:26,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:25:27,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:29,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:29,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:31,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:31,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:33,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:34,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:35,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:36,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:37,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:39,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:40,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:41,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:25:41,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:43,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:45,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:45,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:46,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:47,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:49,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:50,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:51,579 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:25:51,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:51,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:25:53,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:55,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:56,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:25:56,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:25:57,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:25:59,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:01,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:01,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:01,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:03,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:05,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:06,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:06,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:26:06,755 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:26:06,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:26:06,757 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:26:06,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:07,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:07,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:26:09,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:11,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:26:11,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:12,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:13,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:15,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:17,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:17,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:17,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:19,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:21,570 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:26:21,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:22,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:22,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:23,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:25,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:26,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:26:27,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:28,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:29,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:31,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:32,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:33,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:33,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:35,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:37,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:37,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:38,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:39,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:41,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:26:41,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:43,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:44,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:45,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:47,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:47,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:49,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:49,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:51,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:26:51,213 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:26:51,213 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:26:51,215 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:26:51,573 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:26:51,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:51,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:26:52,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:26:53,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:54,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:26:55,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:56,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:26:57,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:26:59,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:00,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:01,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:02,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:03,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:05,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:05,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:07,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:07,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:09,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:10,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:11,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:27:11,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:13,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:15,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:16,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:17,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:17,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:19,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:21,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:21,575 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:27:21,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:22,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:23,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:25,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:26,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:27:26,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:27,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:29,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:31,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:32,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:32,238 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:33,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:35,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:35,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:27:35,836 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:27:35,836 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:27:35,838 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:27:36,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:27:37,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:37,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:37,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:39,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:27:41,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:42,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:43,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:45,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:47,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:47,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:48,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:49,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:51,588 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:27:51,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:52,037 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:27:53,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:53,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:55,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:56,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:27:57,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:27:58,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:27:59,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:01,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:02,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:03,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:04,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:05,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:07,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:07,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:09,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:09,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:11,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:28:11,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:13,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:15,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:15,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:17,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:17,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:19,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:20,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:20,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:28:20,365 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:28:20,366 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:28:20,368 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:28:20,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:28:21,579 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:28:21,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:22,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:23,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:25,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:25,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:26,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:28:27,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:29,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:30,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:31,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:32,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:33,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:35,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:36,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:37,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:37,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:39,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:28:41,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:41,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:43,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:45,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:46,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:47,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:47,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:49,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:51,582 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:28:51,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:52,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:28:52,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:53,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:55,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:56,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:28:57,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:28:57,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:28:59,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:01,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:02,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:03,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:03,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:04,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:29:04,846 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:29:04,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:29:04,848 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:29:05,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:05,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:29:07,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:07,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:08,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:09,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:11,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:29:11,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:13,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:13,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:15,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:17,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:17,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:19,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:19,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:21,585 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:29:21,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:22,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:23,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:24,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:25,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:26,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:29:27,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:29,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:29,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:31,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:32,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:33,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:35,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:35,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:37,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:37,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:39,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:40,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:29:41,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:43,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:45,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:45,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:47,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:47,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:49,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:29:49,531 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:29:49,532 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:29:49,534 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:29:49,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:49,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:29:51,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:51,599 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:29:51,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:52,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:29:53,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:55,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:56,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:29:56,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:29:57,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:29:59,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:01,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:01,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:02,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:03,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:05,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:07,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:07,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:07,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:09,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:11,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:30:11,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:12,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:13,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:15,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:17,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:17,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:17,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:19,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:21,589 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:30:21,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:22,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:23,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:23,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:26,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:26,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:30:28,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:28,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:30,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:32,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:32,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:34,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:34,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:34,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:30:34,079 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:30:34,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:30:34,082 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:30:35,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:30:36,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:37,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:38,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:39,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:40,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:41,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:30:42,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:44,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:44,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:46,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:47,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:48,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:50,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:50,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:51,591 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:30:52,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:52,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:30:54,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:55,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:30:56,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:30:56,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:30:58,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:00,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:00,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:02,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:02,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:04,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:06,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:06,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:07,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:08,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:10,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:11,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:31:11,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:12,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:14,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:16,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:16,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:17,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:18,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:18,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:31:18,639 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:31:18,640 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:31:18,641 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:31:19,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:31:20,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:21,594 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:31:22,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:22,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:22,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:24,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:26,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:26,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:31:27,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:28,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:30,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:32,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:32,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:32,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:34,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:36,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:37,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:38,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:38,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:40,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:41,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:31:42,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:43,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:44,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:46,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:47,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:48,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:48,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:50,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:51,606 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:31:52,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:52,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:31:54,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:54,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:31:56,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:56,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:31:58,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:31:59,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:00,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:02,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:02,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:03,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:32:03,110 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:32:03,110 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:32:03,112 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:32:03,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:32:04,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:04,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:06,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:07,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:08,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:10,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:10,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:11,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:32:12,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:14,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:15,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:16,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:17,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:18,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:20,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:20,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:21,598 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:32:22,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:22,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:24,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:26,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:26,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:26,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:32:28,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:30,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:31,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:32,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:32,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:34,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:36,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:36,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:37,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:38,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:40,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:41,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:32:42,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:42,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:44,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:46,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:47,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:47,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:47,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:32:47,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:32:47,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:32:47,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:32:48,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:48,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:32:50,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:51,600 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:32:52,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:52,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:32:53,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:32:54,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:56,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:56,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:32:58,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:32:58,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:00,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:02,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:02,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:03,665 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:04,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:06,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:07,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:08,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:08,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:10,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:11,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:33:12,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:14,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:14,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:16,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:17,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:18,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:19,692 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:20,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:21,602 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:33:22,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:22,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:24,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:25,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:26,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:26,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:33:28,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:30,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:30,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:32,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:33:32,169 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:33:32,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:33:32,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:33:32,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:32,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:33:32,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:33,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:35,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:36,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:37,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:38,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:40,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:41,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:41,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:33:42,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:44,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:46,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:46,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:47,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:48,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:50,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:51,615 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:33:51,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:52,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:33:52,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:54,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:56,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:33:56,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:33:57,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:33:58,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:00,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:02,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:02,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:02,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:04,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:06,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:07,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:07,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:08,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:10,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:11,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:34:12,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:13,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:14,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:16,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:16,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:34:16,735 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:34:16,736 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:34:16,737 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:34:17,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:17,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:17,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:34:18,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:20,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:21,607 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:34:22,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:22,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:23,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:24,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:26,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:26,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:34:28,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:29,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:30,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:32,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:32,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:34,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:34,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:36,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:37,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:38,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:39,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:40,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:41,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:34:42,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:44,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:45,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:46,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:47,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:48,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:50,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:50,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:51,609 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:34:52,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:34:52,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:54,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:55,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:34:56,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:34:56,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:34:58,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:00,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:01,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:01,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:35:01,300 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:35:01,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:35:01,302 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:35:01,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:35:02,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:02,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:03,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:06,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:06,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:07,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:08,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:10,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:11,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:35:11,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:12,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:14,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:16,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:17,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:17,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:18,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:20,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:21,612 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:35:22,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:22,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:22,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:24,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:26,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:26,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:35:27,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:28,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:30,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:32,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:32,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:33,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:34,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:36,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:37,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:38,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:38,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:40,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:41,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:35:42,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:43,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:44,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:45,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:35:45,719 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:35:45,720 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:35:45,722 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:35:46,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:46,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:35:47,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:47,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:49,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:50,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:51,627 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:35:52,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:35:52,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:54,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:54,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:35:56,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:56,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:35:58,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:35:59,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:00,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:02,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:02,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:04,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:05,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:06,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:07,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:08,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:10,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:10,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:11,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:36:12,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:14,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:16,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:16,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:17,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:18,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:20,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:21,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:21,616 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:36:22,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:22,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:24,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:26,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:26,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:36:26,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:28,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:30,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:36:30,336 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:36:30,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:36:30,338 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:36:30,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:30,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:36:31,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:32,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:32,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:34,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:36,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:37,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:37,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:38,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:40,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:41,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:36:42,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:42,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:43,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:45,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:47,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:47,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:48,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:49,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:51,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:51,619 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:36:52,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:36:53,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:53,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:55,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:56,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:36:57,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:36:58,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:36:59,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:01,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:02,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:03,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:04,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:05,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:07,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:07,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:09,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:09,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:11,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:37:11,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:13,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:14,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:14,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:37:14,910 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:37:14,910 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:37:14,912 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:37:15,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:15,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:37:17,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:17,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:19,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:20,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:21,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:21,621 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:37:22,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:23,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:25,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:25,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:26,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:37:27,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:29,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:30,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:31,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:32,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:33,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:35,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:36,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:37,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:37,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:39,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:41,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:37:41,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:41,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:43,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:45,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:46,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:47,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:47,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:49,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:51,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:51,632 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:37:52,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:52,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:37:53,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:55,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:56,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:37:57,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:37:57,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:59,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:37:59,316 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:37:59,316 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:37:59,318 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:37:59,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:37:59,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:38:01,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:02,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:02,852 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:03,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:05,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:07,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:07,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:08,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:09,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:11,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:38:11,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:13,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:13,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:15,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:17,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:17,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:18,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:19,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:21,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:21,626 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:38:22,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:23,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:24,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:25,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:26,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:38:27,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:29,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:29,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:31,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:32,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:33,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:34,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:35,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:37,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:37,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:39,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:40,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:41,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:38:41,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:43,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:43,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:38:43,864 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:38:43,864 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:38:43,865 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:38:44,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:38:45,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:45,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:47,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:47,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:49,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:50,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:51,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:51,628 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:38:52,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:38:53,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:55,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:56,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:38:56,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:38:57,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:38:59,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:01,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:01,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:02,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:03,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:05,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:06,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:07,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:07,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:09,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:11,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:39:11,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:12,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:13,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:15,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:17,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:17,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:17,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:19,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:21,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:39:21,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:22,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:22,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:23,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:25,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:26,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:39:27,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:28,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:28,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:39:28,341 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:39:28,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:39:28,343 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:39:28,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:39:29,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:31,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:32,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:33,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:33,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:35,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:37,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:37,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:39,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:39,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:41,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:39:41,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:43,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:44,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:45,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:47,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:47,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:49,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:49,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:51,642 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:39:51,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:52,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:39:53,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:55,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:39:55,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:39:57,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:39:59,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:00,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:01,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:02,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:03,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:05,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:05,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:07,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:07,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:09,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:11,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:11,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:40:11,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:13,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:40:13,071 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:40:13,071 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:40:13,073 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:40:13,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:13,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:40:15,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:16,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:17,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:17,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:19,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:21,635 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:40:21,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:21,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:22,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:23,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:25,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:26,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:40:27,253 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:27,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:29,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:31,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:32,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:32,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:33,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:35,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:37,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:37,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:37,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:39,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:41,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:40:41,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:43,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:43,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:45,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:47,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:47,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:48,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:49,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:51,637 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:40:51,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:52,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:40:53,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:53,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:55,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:40:57,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:40:57,567 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:40:57,567 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:40:57,569 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:40:57,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:40:57,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:40:59,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:40:59,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:01,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:02,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:03,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:04,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:05,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:07,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:07,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:09,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:10,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:11,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:41:11,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:13,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:15,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:15,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:17,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:17,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:19,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:20,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:21,639 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:41:21,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:22,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:23,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:25,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:26,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:26,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:41:27,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:29,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:31,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:31,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:32,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:33,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:35,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:36,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:37,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:37,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:39,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:41,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:41:41,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:42,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:42,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:41:42,167 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:41:42,167 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:41:42,169 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:41:42,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:41:43,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:45,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:47,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:47,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:47,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:49,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:51,658 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:41:51,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:52,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:41:52,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:53,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:55,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:56,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:41:57,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:41:58,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:41:59,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:01,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:02,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:03,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:03,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:05,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:07,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:07,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:08,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:09,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:11,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:42:11,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:13,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:14,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:15,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:17,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:17,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:19,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:19,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:21,643 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:42:21,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:22,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:23,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:24,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:25,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:26,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:42:26,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:42:26,587 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:42:26,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:42:26,590 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:42:26,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:42:27,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:29,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:30,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:31,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:32,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:33,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:35,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:35,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:37,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:37,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:39,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:40,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:41,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:42:41,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:43,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:45,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:46,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:47,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:47,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:49,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:51,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:51,646 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:42:51,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:52,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:42:53,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:55,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:56,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:42:56,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:42:57,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:42:59,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:01,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:02,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:02,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:03,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:05,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:07,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:07,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:07,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:09,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:11,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:43:11,027 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:43:11,028 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:43:11,030 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:43:11,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:43:11,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:11,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:43:12,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:13,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:15,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:17,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:18,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:18,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:20,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:21,648 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:43:22,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:22,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:23,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:24,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:26,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:26,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:43:28,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:28,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:30,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:32,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:32,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:34,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:34,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:36,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:37,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:38,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:39,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:40,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:41,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:43:42,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:44,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:44,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:46,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:46,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:48,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:50,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:50,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:51,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:43:51,660 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:43:52,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:54,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:55,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:43:55,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:43:55,556 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:43:55,557 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:43:55,558 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:43:56,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:43:56,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:43:56,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:43:58,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:00,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:00,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:01,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:02,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:04,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:06,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:06,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:06,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:08,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:10,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:11,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:44:11,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:12,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:14,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:16,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:16,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:18,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:20,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:21,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:21,652 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:44:22,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:22,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:24,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:26,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:26,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:44:27,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:28,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:30,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:31,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:32,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:33,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:34,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:36,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:36,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:38,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:38,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:40,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:40,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:44:40,160 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:44:40,160 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:44:40,162 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:44:41,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:44:41,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:44:42,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:43,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:44,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:46,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:46,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:48,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:49,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:50,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:51,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:44:51,655 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:44:52,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:54,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:54,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:44:56,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:56,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:44:58,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:44:59,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:00,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:01,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:02,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:04,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:05,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:06,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:06,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:08,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:10,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:10,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:11,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:45:12,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:14,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:15,836 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:16,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:16,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:18,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:20,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:21,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:21,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:21,657 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:45:22,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:24,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:24,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:45:24,773 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:45:24,773 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:45:24,774 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:45:25,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:45:26,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:26,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:45:26,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:28,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:30,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:31,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:31,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:32,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:34,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:36,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:36,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:37,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:38,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:40,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:41,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:45:42,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:42,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:44,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:46,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:46,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:47,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:48,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:50,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:51,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:45:51,670 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:45:52,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:53,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:45:54,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:56,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:56,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:45:58,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:45:58,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:00,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:01,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:02,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:03,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:04,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:06,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:06,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:08,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:09,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:09,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:46:09,332 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:46:09,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:09,334 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:10,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:10,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:46:10,896 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:46:10,898 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:46:10,899 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:10,902 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:11,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:46:11,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:46:12,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:14,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:16,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:17,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:19,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:21,662 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:46:22,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:24,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:24,976 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry +2024-04-25 23:46:24,977 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:24,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 23:46:24,980 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:24,981 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:24,982 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:24,982 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:24,983 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:24,983 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:24,984 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:24,984 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:24,985 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:24,986 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 23:46:24,986 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 23:46:24,987 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 23:46:25,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 23:46:26,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:26,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:46:28,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 23:46:29,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:32,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:34,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:37,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:39,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:41,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:46:44,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:47,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/config.yaml +2024-04-25 23:46:47,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:51,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:46:51,664 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 23:46:52,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:46:56,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:46:56,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:47:01,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:47:02,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:47:06,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:47:07,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive +2024-04-25 23:47:11,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 23:47:11,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:47:16,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 23:47:17,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: keepalive