diff --git "a/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" "b/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" --- "a/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" +++ "b/wandb/run-20240425_134518-etajcxpg/logs/debug-internal.log" @@ -9518,3 +9518,9379 @@ 2024-04-25 16:17:48,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log 2024-04-25 16:17:49,399 DEBUG SenderThread:156911 [sender.py:send():379] send: stats 2024-04-25 16:17:50,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:51,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:52,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:54,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:54,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:17:54,912 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:17:54,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:17:56,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:56,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:58,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:00,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:02,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:02,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:04,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:06,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:07,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:08,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:18:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:18:09,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:18:10,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:12,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:13,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:14,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:16,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:18,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:18,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:19,401 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:18:20,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:22,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:23,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:24,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:24,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:18:24,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:18:24,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:18:26,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:27,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:18:27,655 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:18:27,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:18:27,657 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:18:28,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:28,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:18:29,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:30,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:32,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:34,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:34,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:36,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:38,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:39,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:18:39,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:18:39,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:18:40,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:40,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:42,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:44,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:45,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:46,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:48,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:49,404 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:18:50,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:51,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:52,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:54,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:54,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:18:54,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:18:54,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:18:56,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:18:56,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:18:58,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:00,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:02,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:02,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:04,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:06,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:07,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:08,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:19:09,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:19:09,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:19:10,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:12,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:13,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:13,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:19:13,145 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:19:13,145 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:19:13,147 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:19:13,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:19:14,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:16,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:18,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:18,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:19,418 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:19:20,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:22,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:24,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:24,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:19:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:19:24,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:19:26,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:28,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:29,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:30,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:32,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:34,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:35,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:36,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:38,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:39,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:19:39,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:19:39,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:19:40,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:40,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:42,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:44,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:45,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:46,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:48,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:49,408 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:19:50,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:50,522 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:52,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:54,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:54,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:19:54,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:19:54,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:19:56,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:19:56,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:58,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:19:58,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:19:58,628 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:19:58,629 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:19:58,630 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:19:59,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:20:00,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:01,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:02,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:04,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:06,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:06,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:08,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:09,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:20:09,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:20:09,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:20:10,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:12,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:12,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:14,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:16,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:17,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:18,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:19,411 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:20:20,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:22,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:23,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:24,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:20:24,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:20:24,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:20:26,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:28,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:28,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:30,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:32,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:34,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:34,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:36,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:38,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:39,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:39,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:20:39,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:20:39,901 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:20:40,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:42,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:44,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:20:44,003 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:20:44,003 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:20:44,005 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:20:44,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:44,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:20:45,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:46,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:48,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:49,412 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:20:50,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:50,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:52,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:54,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:54,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:20:54,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:20:54,972 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:20:56,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:20:56,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:20:58,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:00,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:01,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:02,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:04,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:06,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:06,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:08,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:09,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:21:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:21:09,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:21:10,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:12,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:12,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:14,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:16,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:17,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:18,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:19,415 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:21:20,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:22,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:24,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:24,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:21:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:21:24,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:21:26,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:28,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:28,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:29,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:21:29,406 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:21:29,407 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:21:29,408 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:21:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:21:32,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:33,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:34,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:39,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:39,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:21:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:21:39,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:21:40,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:42,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:44,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:44,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:46,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:48,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:49,417 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:21:50,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:50,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:52,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:54,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:54,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:21:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:21:54,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:21:56,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:21:56,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:21:58,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:00,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:01,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:02,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:04,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:06,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:06,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:08,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:09,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:22:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:22:09,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:22:10,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:12,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:12,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:14,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:14,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:22:14,858 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:22:14,859 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:22:14,860 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:22:15,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:15,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:22:17,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:18,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:19,420 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:22:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:22,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:24,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:24,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:22:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:22:24,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:22:26,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:28,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:28,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:30,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:32,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:34,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:34,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:36,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:38,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:39,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:39,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:22:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:22:39,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:22:40,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:42,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:44,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:44,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:46,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:48,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:49,422 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:22:50,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:50,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:52,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:54,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:54,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:22:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:22:54,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:22:56,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:22:56,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:22:58,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:00,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:23:00,350 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:23:00,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:23:00,353 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:23:00,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:00,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:23:01,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:01,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:04,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:06,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:06,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:08,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:09,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:23:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:23:09,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:23:10,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:12,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:12,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:14,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:16,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:17,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:18,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:19,424 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:23:20,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:22,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:23,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:24,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:24,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:23:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:23:24,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:23:26,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:28,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:28,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:30,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:32,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:34,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:34,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:36,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:38,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:39,501 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:39,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:23:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:23:39,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:23:40,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:42,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:44,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:44,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:45,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:23:45,825 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:23:45,825 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:23:45,827 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:23:46,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:46,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:23:47,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:49,426 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:23:50,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:50,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:52,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:54,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:54,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:23:55,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:23:55,024 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:23:56,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:23:56,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:23:58,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:00,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:01,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:02,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:03,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:05,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:06,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:07,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:09,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:09,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:24:10,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:24:10,018 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:24:11,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:12,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:13,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:15,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:17,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:17,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:19,428 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:24:19,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:21,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:23,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:23,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:24,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:24:25,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:24:25,018 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:24:25,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:27,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:28,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:29,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:31,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:24:31,250 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:24:31,251 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:24:31,252 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:24:31,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:31,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:24:33,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:34,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:35,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:37,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:39,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:39,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:39,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:24:40,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:24:40,018 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:24:41,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:43,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:44,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:45,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:47,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:49,431 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:24:49,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:50,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:51,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:53,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:54,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:24:55,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:24:55,091 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:24:55,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:56,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:24:57,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:24:59,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:01,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:01,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:03,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:05,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:06,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:07,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:09,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:09,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:25:10,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:25:10,077 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:25:11,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:12,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:13,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:15,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:16,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:25:16,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:25:16,786 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:25:16,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:25:17,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:17,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:25:17,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:19,433 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:25:19,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:21,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:23,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:23,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:24,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:25:25,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:25:25,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:25:25,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:27,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:28,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:29,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:31,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:33,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:34,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:35,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:37,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:39,538 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:39,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:39,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:25:40,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:25:40,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:25:41,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:43,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:44,991 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:45,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:47,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:49,435 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:25:49,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:50,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:51,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:53,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:54,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:25:55,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:25:55,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:25:55,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:56,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:25:57,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:25:59,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:01,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:01,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:02,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:26:02,140 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:26:02,141 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:26:02,142 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:26:02,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:26:03,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:05,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:06,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:07,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:09,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:09,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:26:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:26:10,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:26:11,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:12,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:13,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:15,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:17,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:17,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:19,437 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:26:19,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:21,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:23,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:23,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:24,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:26:25,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:26:25,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:26:25,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:27,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:28,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:29,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:31,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:33,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:34,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:35,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:37,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:39,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:39,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:39,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:26:40,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:26:40,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:26:41,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:43,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:44,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:45,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:47,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:26:47,564 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:26:47,564 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:26:47,566 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:26:47,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:47,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:26:49,439 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:26:49,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:50,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:51,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:53,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:54,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:26:55,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:26:55,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:26:55,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:56,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:26:57,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:26:59,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:01,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:01,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:03,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:05,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:06,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:07,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:09,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:09,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:27:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:27:10,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:27:11,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:12,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:13,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:15,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:17,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:17,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:19,441 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:27:19,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:21,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:23,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:24,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:27:25,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:27:25,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:27:25,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:27,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:28,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:29,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:31,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:32,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:27:32,969 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:27:32,970 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:27:32,972 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:27:33,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:34,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:34,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:27:36,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:38,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:39,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:39,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:27:40,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:40,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:27:40,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:27:42,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:44,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:44,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:46,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:48,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:49,455 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:27:50,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:50,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:52,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:54,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:54,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:27:55,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:27:55,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:27:56,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:27:56,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:27:58,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:00,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:01,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:02,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:04,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:06,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:07,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:08,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:09,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:28:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:28:10,078 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:28:10,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:12,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:12,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:14,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:16,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:18,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:18,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:18,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:28:18,366 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:28:18,366 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:28:18,369 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:28:19,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:28:19,445 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:28:20,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:22,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:23,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:24,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:24,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:28:25,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:28:25,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:28:26,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:28,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:29,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:30,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:32,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:34,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:34,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:36,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:38,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:39,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:28:40,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:40,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:28:40,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:28:40,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:42,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:44,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:45,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:46,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:48,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:49,448 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:28:50,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:51,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:52,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:54,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:54,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:28:55,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:28:55,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:28:56,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:28:56,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:28:58,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:00,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:01,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:02,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:03,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:29:03,790 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:29:03,790 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:29:03,792 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:29:04,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:04,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:29:06,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:07,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:08,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:09,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:29:10,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:29:10,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:29:10,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:12,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:12,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:14,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:16,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:18,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:18,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:19,450 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:29:20,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:22,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:23,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:24,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:24,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:29:25,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:29:25,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:29:26,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:28,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:29,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:30,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:32,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:34,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:34,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:36,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:38,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:39,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:29:40,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:40,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:29:40,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:29:40,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:42,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:44,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:45,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:46,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:48,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:49,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:29:49,264 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:29:49,264 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:29:49,266 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:29:49,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:29:50,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:50,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:29:51,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:52,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:54,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:29:55,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:29:55,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:29:56,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:29:56,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:29:58,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:00,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:01,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:02,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:04,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:06,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:07,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:08,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:09,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:30:10,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:30:10,079 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:30:10,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:12,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:12,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:14,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:16,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:18,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:18,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:19,512 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:30:20,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:22,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:23,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:24,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:30:25,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:30:25,080 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:30:26,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:28,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:29,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:30,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:32,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:34,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:34,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:34,581 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:30:34,582 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:30:34,582 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:30:34,584 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:30:35,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:30:36,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:38,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:30:39,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:40,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:30:40,098 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:30:40,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:42,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:44,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:45,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:46,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:48,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:49,514 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:30:50,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:50,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:52,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:54,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:30:55,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:30:55,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:30:56,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:30:56,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:30:58,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:00,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:01,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:02,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:04,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:06,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:07,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:08,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:31:10,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:31:10,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:31:10,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:12,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:12,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:14,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:16,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:18,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:18,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:19,517 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:31:19,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:31:19,896 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:31:19,897 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:31:19,899 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:31:20,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:20,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:31:22,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:23,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:24,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:31:25,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:31:25,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:31:26,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:28,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:28,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:30,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:32,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:34,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:34,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:36,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:38,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:39,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:31:40,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:31:40,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:31:40,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:42,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:44,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:45,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:46,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:48,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:49,530 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:31:50,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:50,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:52,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:54,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:54,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:31:55,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:31:55,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:31:56,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:31:56,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:31:58,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:00,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:01,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:02,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:04,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:05,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:32:05,189 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:32:05,190 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:32:05,192 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:32:05,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:32:06,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:06,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:08,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:09,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:32:10,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:32:10,092 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:32:10,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:12,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:12,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:14,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:16,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:17,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:18,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:19,521 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:32:20,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:22,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:23,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:24,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:24,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:32:25,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:32:25,115 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:32:26,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:28,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:28,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:30,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:32,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:34,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:34,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:36,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:38,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:39,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:32:40,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:32:40,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:32:40,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:42,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:44,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:44,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:46,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:48,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:49,523 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:32:50,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:50,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:50,463 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:32:50,465 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:32:50,465 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:32:50,467 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:32:51,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:32:52,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:54,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:54,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:32:55,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:32:55,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:32:55,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:32:56,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:32:58,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:00,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:01,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:02,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:04,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:06,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:06,726 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:08,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:09,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:33:10,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:33:10,095 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:33:10,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:12,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:12,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:14,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:16,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:17,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:18,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:19,526 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:33:20,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:22,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:22,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:24,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:24,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:33:25,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:33:25,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:33:26,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:28,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:28,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:30,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:32,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:33,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:34,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:35,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:33:35,755 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:33:35,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:33:35,759 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:33:36,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:36,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:33:38,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:39,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:33:40,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:33:40,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:33:40,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:42,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:44,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:44,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:46,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:48,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:49,541 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:33:50,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:50,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:52,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:54,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:54,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:33:55,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:33:55,096 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:33:55,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:33:56,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:33:58,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:00,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:00,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:02,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:04,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:05,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:06,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:08,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:09,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:34:10,168 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:34:10,168 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:34:10,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:11,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:12,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:14,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:16,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:16,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:18,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:19,531 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:34:20,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:20,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:34:20,972 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:34:20,973 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:34:20,975 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:34:21,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:34:21,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:22,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:24,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:24,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:34:25,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:34:25,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:34:26,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:27,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:28,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:30,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:32,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:32,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:34,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:36,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:38,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:38,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:34:40,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:34:40,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:34:40,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:42,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:43,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:44,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:46,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:48,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:49,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:49,533 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:34:50,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:52,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:54,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:54,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:34:54,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:34:55,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:34:55,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:34:56,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:34:58,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:00,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:00,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:02,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:04,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:05,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:06,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:35:06,368 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:35:06,369 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:35:06,371 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:35:06,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:06,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:35:08,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:09,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:35:10,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:35:10,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:35:10,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:11,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:12,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:14,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:16,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:16,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:18,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:19,535 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:35:20,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:21,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:22,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:24,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:24,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:35:25,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:35:25,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:35:26,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:27,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:28,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:30,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:32,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:32,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:34,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:36,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:38,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:38,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:39,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:35:40,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:35:40,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:35:40,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:42,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:43,626 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:44,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:46,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:48,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:49,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:49,548 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:35:50,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:51,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:35:51,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:35:51,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:35:51,694 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:35:52,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:52,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:35:53,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:54,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:35:54,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:35:55,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:35:55,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:35:56,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:58,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:35:59,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:00,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:02,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:04,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:05,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:06,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:08,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:09,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:36:10,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:36:10,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:36:10,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:11,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:12,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:14,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:16,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:16,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:18,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:19,540 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:36:20,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:22,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:22,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:24,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:24,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:36:25,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:36:25,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:36:26,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:27,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:28,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:30,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:32,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:33,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:34,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:36,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:37,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:36:37,007 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:36:37,007 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:36:37,009 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:36:37,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:37,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:36:38,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:39,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:36:40,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:36:40,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:36:40,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:42,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:44,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:44,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:46,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:48,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:49,545 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:36:49,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:50,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:52,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:54,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:54,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:36:55,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:36:55,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:36:55,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:36:56,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:36:58,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:00,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:00,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:02,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:04,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:05,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:06,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:08,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:09,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:37:10,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:37:10,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:37:10,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:11,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:12,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:14,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:16,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:16,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:18,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:19,547 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:37:20,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:22,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:22,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:37:22,172 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:37:22,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:37:22,174 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:37:22,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:22,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:37:23,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:24,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:37:25,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:37:25,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:37:26,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:27,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:28,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:30,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:32,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:32,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:34,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:35,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:37,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:38,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:39,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:39,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:37:40,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:37:40,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:37:41,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:43,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:43,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:45,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:47,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:49,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:49,549 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:37:49,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:51,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:53,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:54,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:37:54,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:37:55,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:37:55,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:37:55,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:57,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:37:59,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:00,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:01,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:03,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:05,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:05,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:07,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:38:07,320 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:38:07,320 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:38:07,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:38:07,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:07,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:38:09,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:09,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:38:10,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:38:10,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:38:10,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:11,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:13,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:15,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:16,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:17,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:19,551 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:38:19,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:21,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:21,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:23,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:24,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:38:25,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:38:25,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:38:25,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:27,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:27,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:29,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:31,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:32,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:33,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:35,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:37,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:38,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:39,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:39,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:38:40,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:38:40,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:38:41,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:43,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:43,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:45,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:47,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:48,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:49,553 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:38:49,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:51,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:52,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:38:52,589 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:38:52,589 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:38:52,591 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:38:52,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:38:53,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:54,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:54,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:38:55,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:38:55,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:38:55,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:57,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:38:59,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:38:59,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:01,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:03,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:05,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:05,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:07,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:09,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:39:09,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:10,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:39:10,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:39:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:11,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:13,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:15,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:16,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:17,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:19,556 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:39:19,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:21,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:21,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:23,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:24,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:39:25,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:39:25,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:39:25,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:26,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:27,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:29,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:31,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:32,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:33,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:35,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:37,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:37,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:39:37,770 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:39:37,771 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:39:37,772 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:39:38,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:38,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:39:39,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:39:40,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:40,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:39:40,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:39:42,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:43,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:44,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:46,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:48,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:48,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:49,557 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:39:50,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:52,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:54,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:54,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:39:54,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:39:55,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:39:55,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:39:56,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:58,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:39:59,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:00,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:02,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:04,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:04,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:06,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:08,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:09,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:40:10,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:10,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:40:10,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:40:10,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:12,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:14,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:15,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:16,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:18,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:19,571 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:40:20,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:21,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:22,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:23,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:40:23,105 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:40:23,105 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:40:23,107 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:40:24,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:24,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:40:24,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:40:25,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:40:25,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:40:26,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:26,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:28,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:30,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:32,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:32,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:34,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:36,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:37,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:38,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:39,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:40:40,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:40,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:40:40,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:40:42,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:42,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:44,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:46,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:48,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:48,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:49,562 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:40:50,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:52,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:53,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:40:54,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:54,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:40:55,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:40:55,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:40:56,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:58,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:40:59,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:00,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:02,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:04,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:04,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:06,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:08,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:41:08,286 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:41:08,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:41:08,288 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:41:09,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:41:09,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:41:10,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:10,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:10,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:41:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:41:12,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:14,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:15,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:16,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:18,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:19,564 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:41:20,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:20,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:22,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:24,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:24,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:41:25,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:41:25,162 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:41:26,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:26,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:28,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:30,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:31,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:32,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:34,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:36,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:37,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:38,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:39,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:41:40,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:40,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:41:40,162 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:41:42,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:42,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:44,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:46,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:47,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:48,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:49,567 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:41:50,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:52,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:53,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:41:53,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:41:53,439 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:41:53,439 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:41:53,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:41:54,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:54,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:41:54,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:41:55,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:41:55,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:41:56,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:58,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:41:58,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:00,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:02,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:04,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:04,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:06,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:08,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:09,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:09,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:42:10,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:10,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:42:10,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:42:12,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:14,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:15,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:16,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:18,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:19,580 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:42:20,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:20,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:22,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:24,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:24,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:42:25,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:42:25,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:42:25,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:26,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:28,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:30,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:31,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:32,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:34,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:36,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:36,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:38,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:38,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:42:38,544 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:42:38,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:42:38,545 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:42:39,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:42:39,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:42:40,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:42:40,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:42:40,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:42,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:42,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:44,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:46,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:47,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:48,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:49,571 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:42:50,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:52,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:52,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:42:54,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:54,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:42:55,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:42:55,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:42:56,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:58,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:42:58,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:00,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:02,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:03,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:04,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:06,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:08,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:09,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:09,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:43:10,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:43:10,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:43:10,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:12,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:14,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:14,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:16,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:18,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:19,573 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:43:19,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:20,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:22,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:23,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:43:23,606 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:43:23,606 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:43:23,608 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:43:24,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:24,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:43:24,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:24,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:43:25,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:43:25,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:43:26,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:28,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:30,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:30,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:32,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:34,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:35,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:36,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:38,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:39,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:43:40,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:43:40,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:43:40,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:41,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:42,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:44,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:46,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:46,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:48,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:49,576 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:43:50,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:51,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:52,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:54,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:54,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:43:55,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:43:55,163 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:43:56,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:43:57,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:43:58,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:00,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:02,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:02,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:04,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:06,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:07,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:08,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:08,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:44:08,744 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:44:08,744 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:44:08,747 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:44:09,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:44:09,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:44:10,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:44:10,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:44:10,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:12,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:13,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:14,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:16,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:18,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:18,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:19,588 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:44:20,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:22,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:24,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:24,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:24,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:44:25,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:44:25,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:44:26,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:28,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:29,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:30,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:32,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:34,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:34,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:36,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:38,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:39,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:44:40,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:44:40,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:44:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:40,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:42,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:44,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:45,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:46,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:48,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:49,580 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:44:50,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:51,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:52,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:53,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:44:53,851 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:44:53,852 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:44:53,853 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:44:54,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:54,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:44:54,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:44:55,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:44:55,164 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:44:56,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:44:56,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:44:58,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:00,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:02,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:02,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:04,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:06,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:07,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:08,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:45:10,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:45:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:45:10,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:12,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:12,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:14,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:16,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:18,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:18,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:19,582 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:45:20,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:22,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:23,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:24,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:24,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:45:25,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:45:25,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:45:26,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:28,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:29,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:30,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:32,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:34,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:34,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:36,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:38,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:38,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:45:38,911 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:45:38,912 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:45:38,913 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:45:39,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:45:39,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:45:40,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:45:40,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:45:40,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:42,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:44,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:45,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:46,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:48,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:49,585 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:45:50,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:50,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:52,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:54,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:54,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:45:55,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:45:55,209 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:45:56,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:45:56,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:45:58,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:00,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:01,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:02,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:04,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:06,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:06,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:08,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:46:10,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:46:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:46:10,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:12,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:12,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:14,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:16,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:17,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:18,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:19,598 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:46:20,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:22,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:23,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:24,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:46:24,041 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:46:24,042 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:46:24,043 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:46:24,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:24,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:46:24,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:46:25,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:46:25,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:46:26,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:28,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:28,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:30,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:32,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:34,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:34,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:36,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:38,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:39,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:46:40,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:46:40,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:46:40,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:42,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:44,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:44,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:46,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:48,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:49,589 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:46:50,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:50,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:52,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:54,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:54,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:46:55,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:46:55,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:46:56,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:46:56,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:46:58,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:00,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:01,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:02,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:04,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:06,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:07,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:08,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:09,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:47:09,108 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:47:09,108 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:47:09,110 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:47:09,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:47:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:47:10,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:47:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:47:10,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:12,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:12,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:14,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:16,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:18,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:18,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:19,591 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:47:20,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:22,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:23,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:24,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:24,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:47:25,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:47:25,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:47:26,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:28,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:28,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:30,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:32,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:34,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:34,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:36,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:38,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:39,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:47:40,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:47:40,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:47:40,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:42,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:44,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:45,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:46,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:48,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:49,593 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:47:50,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:50,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:52,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:54,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:47:54,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:47:54,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:47:54,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:47:54,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:54,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:47:54,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:47:55,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:47:55,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:47:55,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:47:55,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:47:58,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:00,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:01,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:02,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:04,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:06,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:06,821 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:08,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:09,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:48:10,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:48:10,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:48:10,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:12,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:12,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:14,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:16,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:17,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:18,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:19,606 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:48:20,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:22,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:23,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:24,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:24,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:48:25,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:48:25,170 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:48:26,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:28,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:28,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:30,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:32,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:33,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:34,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:36,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:38,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:39,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:39,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:48:39,304 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:48:39,304 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:48:39,306 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:48:39,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:39,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:48:39,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:48:40,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:48:40,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:48:42,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:44,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:44,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:46,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:48,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:49,598 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:48:50,139 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:50,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:52,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:54,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:54,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:48:55,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:48:55,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:48:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:48:56,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:48:58,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:00,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:00,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:02,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:04,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:06,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:06,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:08,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:09,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:49:10,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:49:10,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:49:10,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:11,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:12,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:14,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:16,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:17,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:18,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:19,600 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:49:20,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:22,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:22,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:24,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:49:24,426 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:49:24,427 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:49:24,428 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:49:24,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:24,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:49:24,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:49:25,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:49:25,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:49:25,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:28,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:28,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:30,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:32,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:33,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:34,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:36,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:38,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:38,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:39,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:49:40,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:49:40,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:49:40,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:42,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:44,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:44,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:46,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:48,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:49,603 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:49:49,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:50,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:52,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:54,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:54,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:49:55,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:49:55,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:49:55,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:49:56,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:49:58,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:00,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:00,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:02,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:04,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:06,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:06,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:08,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:09,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:50:09,675 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:50:09,675 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:50:09,677 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:50:09,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:50:09,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:50:10,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:50:10,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:50:10,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:11,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:11,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:12,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:14,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:16,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:16,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:18,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:19,616 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:50:20,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:22,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:22,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:24,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:24,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:50:25,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:50:25,171 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:50:26,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:27,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:28,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:30,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:32,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:33,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:34,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:36,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:38,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:38,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:39,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:50:40,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:50:40,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:50:40,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:42,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:44,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:44,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:46,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:48,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:49,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:49,607 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:50:50,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:52,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:54,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:50:54,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:50:54,914 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:50:54,915 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:50:54,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:50:54,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:54,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:50:54,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:50:55,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:50:55,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:50:55,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:56,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:50:58,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:00,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:00,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:02,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:04,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:05,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:06,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:08,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:09,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:51:10,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:51:10,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:51:10,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:11,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:12,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:14,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:16,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:16,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:18,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:19,609 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:51:20,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:22,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:22,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:24,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:24,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:51:25,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:51:25,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:51:26,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:27,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:28,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:30,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:32,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:32,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:34,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:36,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:38,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:38,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:39,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:51:40,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:51:40,099 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:51:40,099 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:51:40,101 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:51:40,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:51:40,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:51:40,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:40,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:51:41,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:42,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:43,663 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:44,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:46,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:48,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:49,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:49,611 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:51:50,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:52,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:54,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:51:54,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:51:54,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:55,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:51:55,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:51:56,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:58,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:51:59,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:00,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:03,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:05,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:05,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:07,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:09,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:09,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:52:10,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:52:10,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:52:10,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:11,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:13,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:15,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:16,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:17,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:19,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:19,625 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:52:21,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:21,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:23,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:24,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:52:25,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:25,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:52:25,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:52:25,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:52:25,318 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:52:25,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:52:25,319 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:52:26,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:26,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:52:27,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:27,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:29,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:31,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:32,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:33,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:35,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:37,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:37,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:39,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:39,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:52:40,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:52:40,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:52:41,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:43,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:43,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:45,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:47,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:48,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:49,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:49,615 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:52:51,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:53,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:54,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:54,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:52:54,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:52:55,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:52:55,172 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:52:56,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:58,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:52:59,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:00,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:02,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:04,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:05,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:06,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:08,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:09,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:53:10,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:10,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:53:10,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:53:10,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:10,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:53:10,488 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:53:10,489 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:53:10,491 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:53:11,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:53:12,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:14,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:15,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:16,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:18,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:19,617 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:53:20,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:21,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:22,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:24,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:24,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:53:25,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:53:25,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:53:26,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:26,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:28,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:30,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:32,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:32,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:34,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:36,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:37,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:38,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:39,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:53:40,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:40,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:53:40,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:53:42,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:42,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:44,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:46,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:48,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:48,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:49,620 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:53:50,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:52,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:53,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:53:54,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:54,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:53:55,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:53:55,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:53:55,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:53:55,654 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:53:55,655 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:53:55,657 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:53:56,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:56,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:53:58,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:53:59,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:00,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:02,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:04,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:04,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:06,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:08,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:09,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:54:10,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:10,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:10,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:54:10,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:54:12,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:14,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:15,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:16,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:18,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:19,622 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:54:20,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:20,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:22,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:24,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:24,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:54:25,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:54:25,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:54:26,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:26,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:28,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:30,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:31,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:32,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:34,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:36,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:37,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:38,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:39,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:54:40,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:40,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:54:40,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:54:40,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:54:40,893 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:54:40,893 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:54:40,896 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:54:41,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:54:42,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:42,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:44,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:46,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:48,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:48,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:49,624 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:54:50,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:52,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:53,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:54:54,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:54,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:54:55,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:54:55,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:54:56,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:58,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:54:58,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:00,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:02,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:04,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:04,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:06,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:08,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:09,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:09,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:55:10,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:55:10,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:55:10,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:12,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:14,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:15,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:16,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:18,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:19,626 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:55:20,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:20,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:22,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:24,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:24,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:55:25,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:55:25,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:55:26,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:26,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:55:26,033 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:55:26,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:55:26,035 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:55:26,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:26,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:55:28,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:30,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:31,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:32,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:34,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:36,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:36,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:38,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:39,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:55:40,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:55:40,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:55:40,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:42,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:42,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:44,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:46,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:47,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:48,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:49,628 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:55:50,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:52,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:53,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:55:54,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:54,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:55:55,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:55:55,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:55:56,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:58,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:55:58,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:00,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:02,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:03,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:04,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:06,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:08,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:09,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:09,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:56:10,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:56:10,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:56:10,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:11,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:56:11,064 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:56:11,064 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:56:11,066 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:56:11,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:56:12,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:14,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:14,651 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:16,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:18,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:19,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:56:20,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:20,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:22,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:24,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:24,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:56:25,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:56:25,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:56:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:26,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:28,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:30,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:30,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:32,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:34,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:36,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:36,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:38,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:40,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:56:40,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:56:40,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:56:40,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:41,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:42,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:44,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:46,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:47,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:48,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:49,633 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:56:50,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:52,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:52,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:54,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:55,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:56:55,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:56:55,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:56:56,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:56:56,242 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:56:56,242 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:56:56,244 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:56:56,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:56:56,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:56:58,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:56:58,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:00,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:02,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:03,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:04,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:06,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:08,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:08,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:10,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:57:10,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:57:10,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:57:10,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:12,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:14,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:14,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:16,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:18,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:19,636 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:57:19,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:20,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:22,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:24,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:25,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:57:25,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:57:25,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:57:26,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:28,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:30,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:30,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:32,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:34,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:35,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:36,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:38,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:40,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:57:40,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:57:40,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:57:40,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:41,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:41,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:57:41,359 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:57:41,359 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:57:41,360 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:57:41,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:57:42,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:44,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:46,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:46,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:48,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:49,639 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:57:50,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:52,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:52,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:54,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:55,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:57:55,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:57:55,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:57:56,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:57:57,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:57:58,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:00,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:02,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:03,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:04,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:06,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:08,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:08,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:10,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:58:10,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:58:10,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:58:10,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:12,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:13,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:14,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:16,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:18,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:19,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:19,641 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:58:20,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:22,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:24,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:24,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:25,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:58:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:58:25,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:58:26,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:26,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:58:26,656 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:58:26,657 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:58:26,658 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:58:27,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:58:28,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:30,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:30,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:32,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:34,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:35,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:36,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:38,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:40,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:58:40,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:58:40,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:58:40,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:41,091 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:42,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:44,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:46,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:46,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:48,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:49,643 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:58:50,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:51,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:52,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:54,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:55,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:58:55,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:58:55,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:58:56,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:58:57,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:58:58,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:00,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:02,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:02,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:04,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:06,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:08,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:08,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:10,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:59:10,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:59:10,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:59:10,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:11,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:59:11,863 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:59:11,863 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:59:11,864 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:59:12,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:12,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:59:13,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:14,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:16,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:18,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:19,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:19,645 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:59:20,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:22,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:24,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:24,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:25,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:59:25,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:59:25,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:59:26,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:28,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:29,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:30,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:32,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:34,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:35,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:36,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:38,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:40,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:59:40,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:59:40,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:59:40,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:40,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:42,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:44,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:45,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:46,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:48,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:49,647 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:59:50,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:50,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:52,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:54,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:55,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:59:55,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:59:55,213 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:59:56,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:59:56,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:59:56,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:59:56,945 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:59:56,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:59:56,947 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:59:57,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:59:58,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:00,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:01,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:02,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:04,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:06,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:06,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:08,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:10,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:00:10,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:00:10,193 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:00:10,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:12,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:12,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:14,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:16,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:17,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:18,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:19,650 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:00:20,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:22,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:23,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:24,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:25,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:00:25,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:00:25,193 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:00:26,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:28,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:28,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:30,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:32,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:33,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:34,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:36,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:38,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:39,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:40,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:00:40,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:00:40,225 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:00:40,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:42,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:00:42,026 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:00:42,026 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:00:42,028 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:00:42,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:42,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:00:43,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:44,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:44,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:46,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:48,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:49,652 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:00:50,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:50,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:52,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:54,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:55,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:00:55,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:00:55,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:00:56,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:00:56,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:00:58,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:00,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:01,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:02,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:04,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:06,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:07,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:08,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:10,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:01:10,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:01:10,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:01:10,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:12,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:12,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:14,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:16,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:18,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:18,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:19,654 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:01:20,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:22,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:23,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:24,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:25,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:01:25,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:01:25,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:01:26,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:27,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:01:27,116 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:01:27,116 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:01:27,118 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:01:27,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:27,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:01:28,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:28,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:30,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:32,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:34,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:34,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:36,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:38,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:39,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:40,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:01:40,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:01:40,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:01:40,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:42,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:44,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:45,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:46,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:48,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:49,656 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:01:50,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:50,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:52,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:54,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:55,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:01:55,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:01:55,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:01:56,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:01:56,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:01:58,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:00,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:01,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:02,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:04,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:06,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:06,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:08,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:10,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:02:10,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:02:10,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:02:10,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:12,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:12,228 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:02:12,230 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:02:12,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:02:12,232 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:02:12,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:12,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:02:13,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:14,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:16,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:17,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:18,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:19,659 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:02:20,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:22,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:22,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:24,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:25,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:02:25,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:02:25,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:02:26,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:28,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:28,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:30,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:32,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:33,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:34,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:36,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:38,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:39,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:40,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:02:40,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:02:40,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:02:40,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:42,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:44,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:44,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:46,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:48,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:49,672 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:02:49,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:50,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:52,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:54,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:55,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:02:55,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:02:55,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:02:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:02:56,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:57,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:02:57,266 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:02:57,266 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:02:57,267 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:02:57,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:02:57,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:02:58,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:00,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:00,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:02,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:04,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:06,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:06,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:08,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:10,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:03:10,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:03:10,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:03:10,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:11,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:12,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:14,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:16,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:16,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:18,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:19,663 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:03:20,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:22,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:22,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:24,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:25,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:03:25,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:03:25,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:03:26,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:27,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:28,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:30,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:32,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:33,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:34,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:36,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:38,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:38,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:40,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:03:40,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:03:40,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:03:40,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:42,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:03:42,250 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:03:42,250 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:03:42,252 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:03:42,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:42,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:03:43,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:44,025 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:44,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:46,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:48,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:49,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:49,665 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:03:50,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:52,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:54,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:03:54,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:55,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:03:55,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:03:55,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:03:56,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:03:58,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:00,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:00,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:02,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:05,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:05,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:07,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:09,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:10,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:04:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:04:10,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:04:11,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:11,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:13,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:15,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:16,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:17,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:19,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:19,668 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:04:21,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:21,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:23,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:25,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:04:25,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:25,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:04:25,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:04:27,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:27,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:27,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:04:27,312 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:04:27,312 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:04:27,314 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:04:28,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:28,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:04:29,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:31,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:32,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:33,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:35,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:37,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:38,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:39,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:40,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:04:40,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:04:40,195 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:04:41,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:43,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:43,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:45,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:47,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:48,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:49,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:49,680 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:04:51,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:53,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:54,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:04:55,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:04:55,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:55,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:04:55,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:04:57,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:59,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:04:59,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:01,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:03,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:05,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:05,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:07,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:09,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:10,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:05:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:05:10,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:05:10,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:11,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:12,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:05:12,351 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:05:12,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:05:12,354 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:05:13,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:13,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:05:14,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:15,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:15,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:17,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:19,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:19,671 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:05:21,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:21,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:23,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:25,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:05:25,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:25,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:05:25,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:05:26,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:27,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:29,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:31,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:32,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:33,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:35,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:37,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:37,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:39,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:40,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:05:40,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:05:40,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:05:41,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:42,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:43,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:45,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:47,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:48,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:49,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:49,674 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:05:51,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:53,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:53,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:05:55,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:05:55,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:55,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:05:55,196 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:05:57,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:57,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:05:57,440 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:05:57,440 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:05:57,441 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:05:58,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:58,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:05:59,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:05:59,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:01,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:03,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:04,664 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:05,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:07,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:09,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:10,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:06:10,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:10,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:06:10,221 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:06:11,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:13,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:15,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:15,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:17,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:19,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:19,677 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:06:20,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:21,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:23,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:25,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:06:25,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:25,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:06:25,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:06:26,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:27,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:29,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:31,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:31,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:33,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:35,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:37,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:37,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:39,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:40,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:06:40,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:06:40,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:06:41,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:42,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:42,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:06:42,753 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:06:42,753 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:06:42,756 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:06:43,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:43,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:06:44,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:45,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:47,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:48,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:49,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:49,689 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:06:51,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:53,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:53,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:55,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:06:55,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:55,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:06:55,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:06:57,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:06:59,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:06:59,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:01,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:03,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:04,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:05,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:07,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:09,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:09,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:10,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:07:10,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:07:10,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:07:11,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:13,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:15,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:15,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:17,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:19,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:19,681 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:07:20,682 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:21,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:23,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:25,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:07:25,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:07:25,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:07:25,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:26,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:27,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:28,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:07:28,186 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:07:28,187 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:07:28,188 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:07:28,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:07:29,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:30,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:31,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:31,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:32,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:34,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:36,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:37,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:38,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:40,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:07:40,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:07:40,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:07:40,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:42,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:42,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:44,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:46,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:48,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:48,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:49,683 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:07:50,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:52,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:53,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:07:54,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:55,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:07:55,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:07:55,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:07:56,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:58,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:07:58,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:00,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:02,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:04,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:04,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:06,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:08,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:09,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:10,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:08:10,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:08:10,214 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:08:10,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:12,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:13,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:08:13,478 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:08:13,478 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:08:13,480 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:08:14,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:14,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:08:15,262 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:16,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:18,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:19,685 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:08:20,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:20,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:22,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:24,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:25,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:08:25,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:08:25,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:08:26,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:26,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:28,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:30,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:31,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:32,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:34,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:36,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:36,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:38,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:40,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:08:40,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:08:40,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:08:40,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:42,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:42,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:44,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:46,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:47,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:48,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:49,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:08:50,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:52,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:53,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:54,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:55,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:08:55,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:08:55,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:08:56,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:58,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:08:58,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:08:58,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:08:58,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:08:58,692 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:08:58,694 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:08:59,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:09:00,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:02,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:04,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:04,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:06,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:08,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:09,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:10,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:09:10,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:09:10,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:09:10,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:12,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:14,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:14,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:16,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:18,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:19,690 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:09:20,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:20,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:22,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:24,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:25,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:09:25,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:09:25,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:09:25,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:26,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:28,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:30,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:30,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:32,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:34,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:35,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:36,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:38,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:40,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:09:40,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:09:40,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:09:40,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:41,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:42,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:43,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:09:43,935 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:09:43,936 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:09:43,937 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:09:44,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:44,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:09:46,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:46,712 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:48,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:49,692 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:09:50,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:52,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:52,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:54,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:09:55,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:09:55,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:09:56,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:09:57,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:09:58,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:00,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:02,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:02,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:04,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:06,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:08,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:08,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:10,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:10:10,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:10:10,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:10:10,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:12,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:13,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:14,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:16,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:18,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:19,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:19,694 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:10:20,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:22,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:24,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:24,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:25,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:10:25,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:10:25,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:10:26,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:28,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:29,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:10:29,132 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:10:29,133 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:10:29,134 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:10:29,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:10:30,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:30,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:32,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:34,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:35,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:36,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:38,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:40,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:10:40,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:10:40,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:10:40,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:41,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:42,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:44,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:46,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:47,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:48,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:49,697 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:10:50,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:52,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:52,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:54,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:10:55,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:10:55,215 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:10:56,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:10:57,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:10:58,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:00,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:02,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:03,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:04,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:06,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:08,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:08,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:10,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:11:10,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:11:10,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:11:10,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:12,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:14,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:14,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:11:14,277 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:11:14,278 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:11:14,279 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:11:14,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:14,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:11:16,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:18,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:19,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:19,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:11:20,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:22,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:24,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:25,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:25,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:11:25,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:11:25,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:11:26,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:28,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:30,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:30,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:32,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:34,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:35,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:36,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:38,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:40,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:11:40,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:11:40,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:11:40,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:41,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:42,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:44,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:46,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:46,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:48,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:49,701 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:11:50,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:52,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:52,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:54,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:55,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:11:55,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:11:55,216 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:11:56,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:57,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:11:58,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:11:59,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:11:59,430 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:11:59,430 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:11:59,432 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:11:59,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:12:00,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:02,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:03,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:04,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:06,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:08,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:08,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:10,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:12:10,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:12:10,288 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:12:10,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:12,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:13,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:14,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:16,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:18,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:19,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:19,704 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:12:20,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:22,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:24,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:24,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:25,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:12:25,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:12:25,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:12:26,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:28,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:30,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:30,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:32,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:34,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:35,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:36,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:38,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:40,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:12:40,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:12:40,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:12:40,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:40,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:42,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:44,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:12:44,531 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:12:44,531 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:12:44,534 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:12:44,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:44,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:12:46,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:46,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:48,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:49,706 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:12:50,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:51,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:52,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:54,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:12:55,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:12:55,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:12:56,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:12:57,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:12:58,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:00,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:02,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:02,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:04,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:06,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:07,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:08,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:10,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:13:10,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:13:10,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:13:10,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:12,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:13,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:14,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:16,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:18,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:18,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:19,708 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:13:20,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:22,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:24,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:24,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:25,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:13:25,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:13:25,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:13:26,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:28,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:29,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:29,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:13:29,634 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:13:29,634 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:13:29,636 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:13:29,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:13:30,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:32,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:34,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:35,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:36,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:38,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:40,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:13:40,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:40,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:13:40,268 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:13:40,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:42,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:44,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:45,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:46,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:48,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:49,710 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:13:50,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:51,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:52,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:54,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:13:55,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:13:55,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:13:56,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:13:56,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:13:58,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:00,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:02,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:02,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:04,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:06,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:07,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:08,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:10,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:14:10,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:14:10,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:14:10,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:12,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:12,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:14,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:14:14,746 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:14:14,746 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:14:14,748 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:14:14,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:14,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:14:16,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:18,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:18,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:19,712 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:14:20,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:22,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:23,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:24,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:25,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:14:25,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:14:25,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:14:26,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:28,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:29,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:30,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:32,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:34,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:34,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:36,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:38,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:39,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:40,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:14:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:14:40,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:14:40,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:42,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:44,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:45,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:46,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:48,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:49,715 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:14:50,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:50,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:52,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:54,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:14:55,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:14:55,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:14:56,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:14:56,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:58,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:14:59,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:14:59,826 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:14:59,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:14:59,828 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:14:59,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:15:00,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:01,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:01,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:02,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:04,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:06,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:06,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:08,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:10,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:15:10,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:15:10,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:15:10,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:12,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:12,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:14,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:16,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:17,808 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:18,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:19,717 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:15:20,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:22,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:23,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:24,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:25,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:15:25,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:15:25,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:15:26,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:28,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:28,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:30,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:32,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:34,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:34,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:36,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:38,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:39,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:40,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:15:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:15:40,269 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:15:40,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:42,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:44,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:44,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:15:44,874 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:15:44,875 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:15:44,878 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:15:44,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:44,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:15:45,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:46,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:48,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:49,719 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:15:50,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:50,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:52,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:54,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:15:55,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:15:55,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:15:55,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:15:56,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:15:58,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:00,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:00,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:02,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:04,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:06,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:06,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:08,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:10,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:16:10,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:16:10,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:16:10,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:11,766 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:12,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:14,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:16,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:17,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:18,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:19,721 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:16:20,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:22,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:22,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:24,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:25,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:16:25,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:16:25,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:16:26,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:27,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:28,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:29,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:16:29,805 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:16:29,806 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:16:29,808 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:16:29,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:16:30,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:31,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:32,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:33,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:34,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:36,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:38,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:39,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:40,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:16:40,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:16:40,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:16:41,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:43,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:44,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:45,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:47,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:49,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:49,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:49,724 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:16:51,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:53,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:55,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:55,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:16:55,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:16:55,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:16:55,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:16:57,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:16:59,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:00,476 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:01,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:03,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:05,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:05,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:07,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:09,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:10,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:17:10,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:17:10,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:17:11,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:11,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:13,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:14,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:17:14,902 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:17:14,902 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:17:14,904 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:17:15,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:15,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:17:16,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:16,668 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:17,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:19,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:19,726 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:17:21,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:22,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:23,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:25,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:25,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:17:25,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:17:25,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:17:27,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:27,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:29,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:31,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:32,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:33,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:35,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:37,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:38,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:39,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:40,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:17:40,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:17:40,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:17:41,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:43,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:43,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:45,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:47,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:49,081 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:49,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:49,728 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:17:51,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:53,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:54,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:55,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:17:55,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:55,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:17:55,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:17:57,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:59,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:17:59,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:17:59,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:17:59,920 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:17:59,920 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:17:59,921 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:18:00,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:18:01,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:02,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:03,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:05,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:05,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:07,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:09,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:10,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:18:10,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:18:10,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:18:10,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:11,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:13,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:15,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:16,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:17,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:19,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:19,729 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:18:21,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:21,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:23,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:25,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:18:25,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:18:25,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:18:25,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:26,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:27,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:29,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:31,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:32,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:33,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:35,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:37,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:37,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:39,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:40,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:18:40,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:18:40,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:18:41,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:42,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:43,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:44,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:18:44,804 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:18:44,805 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:18:44,807 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:18:45,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:45,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:18:46,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:47,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:48,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:49,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:49,732 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:18:51,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:53,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:53,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:55,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:18:55,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:18:55,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:18:55,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:57,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:18:59,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:18:59,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:01,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:03,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:04,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:05,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:07,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:09,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:09,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:10,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:19:10,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:19:10,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:19:11,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:13,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:15,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:15,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:17,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:19,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:19,746 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:19:20,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:21,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:23,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:25,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:19:25,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:19:25,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:19:25,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:26,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:27,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:29,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:29,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:19:29,817 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:19:29,817 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:19:29,819 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:19:30,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:19:31,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:31,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:32,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:33,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:35,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:37,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:37,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:39,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:40,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:19:40,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:19:40,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:19:41,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:42,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:43,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:45,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:47,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:47,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:49,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:49,737 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:19:51,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:53,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:53,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:55,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:19:55,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:19:55,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:19:55,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:57,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:19:58,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:19:59,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:01,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:03,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:03,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:05,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:07,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:09,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:09,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:10,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:20:10,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:20:10,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:20:11,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:13,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:14,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:14,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:20:14,731 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:20:14,731 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:20:14,733 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:20:15,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:15,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:20:16,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:17,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:19,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:19,739 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:20:19,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:21,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:23,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:25,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:20:25,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:25,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:20:25,271 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:20:25,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:27,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:29,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:30,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:31,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:33,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:35,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:36,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:37,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:39,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:40,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:20:40,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:20:40,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:20:41,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:41,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:43,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:45,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:47,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:47,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:49,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:49,741 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:20:51,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:52,672 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:53,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:55,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:20:55,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:20:55,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:20:55,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:57,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:58,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:20:59,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:20:59,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:20:59,905 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:20:59,906 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:20:59,907 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:21:00,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:21:01,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:02,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:03,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:03,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:05,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:07,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:08,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:09,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:10,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:21:10,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:21:10,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:21:11,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:13,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:14,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:15,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:17,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:19,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:19,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:19,755 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:21:21,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:23,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:25,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:25,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:21:25,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:21:25,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:21:25,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:27,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:29,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:30,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:31,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:33,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:35,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:35,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:37,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:39,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:40,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:21:40,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:21:40,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:21:41,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:41,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:43,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:45,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:21:45,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:21:45,014 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:21:45,016 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:21:45,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:45,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:21:46,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:46,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:47,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:49,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:49,745 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:21:51,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:52,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:53,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:55,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:21:55,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:21:55,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:21:55,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:57,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:21:57,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:21:59,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:01,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:03,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:03,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:05,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:07,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:08,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:09,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:10,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:22:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:22:10,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:22:11,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:13,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:13,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:15,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:17,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:19,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:19,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:19,747 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:22:21,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:23,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:24,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:25,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:22:25,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:22:25,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:22:25,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:27,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:29,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:30,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:30,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:22:30,164 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:22:30,165 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:22:30,166 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:22:30,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:22:31,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:32,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:33,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:35,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:35,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:37,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:39,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:40,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:22:40,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:22:40,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:22:40,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:41,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:43,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:45,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:46,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:47,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:49,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:49,749 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:22:51,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:51,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:53,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:55,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:22:55,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:22:55,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:22:55,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:57,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:22:57,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:22:59,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:01,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:02,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:03,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:05,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:06,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:07,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:08,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:23:10,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:23:10,272 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:23:10,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:12,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:13,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:14,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:15,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:23:15,240 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:23:15,241 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:23:15,242 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:23:15,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:23:16,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:18,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:18,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:19,762 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:23:20,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:22,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:24,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:24,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:25,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:23:25,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:23:25,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:23:26,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:28,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:29,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:30,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:32,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:34,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:34,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:36,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:38,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:40,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:40,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:23:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:23:40,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:23:40,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:42,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:44,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:45,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:46,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:48,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:49,754 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:23:50,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:51,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:52,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:54,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:55,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:23:55,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:23:55,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:23:56,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:23:56,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:23:58,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:00,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:24:00,241 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:24:00,241 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:24:00,243 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:24:00,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:00,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:24:02,013 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:02,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:04,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:06,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:07,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:08,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:24:10,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:24:10,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:24:10,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:12,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:12,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:14,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:16,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:18,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:18,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:19,757 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:24:20,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:22,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:23,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:24,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:25,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:24:25,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:24:25,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:24:26,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:28,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:28,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:30,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:32,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:34,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:34,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:36,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:38,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:39,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:40,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:24:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:24:40,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:24:40,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:42,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:44,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:45,215 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:45,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:24:45,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:24:45,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:24:45,249 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:24:45,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:24:46,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:48,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:49,759 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:24:50,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:50,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:52,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:54,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:55,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:24:55,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:24:55,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:24:56,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:24:56,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:24:58,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:00,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:01,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:02,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:04,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:06,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:06,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:08,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:25:10,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:25:10,273 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:25:10,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:12,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:12,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:14,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:16,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:17,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:18,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:19,770 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:25:20,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:22,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:23,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:24,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:25,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:25:25,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:25:25,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:25:26,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:28,428 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:28,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:30,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:25:30,260 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:25:30,261 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:25:30,262 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:25:30,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:30,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:25:32,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:33,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:34,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:36,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:38,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:39,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:40,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:25:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:25:40,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:25:40,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:42,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:44,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:44,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:46,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:48,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:49,763 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:25:49,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:50,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:52,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:54,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:54,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:25:55,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:25:55,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:25:55,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:25:56,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:25:58,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:00,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:00,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:02,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:04,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:05,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:06,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:08,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:10,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:26:10,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:26:10,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:26:10,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:11,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:12,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:14,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:15,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:26:15,432 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:26:15,432 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:26:15,435 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:26:15,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:26:16,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:16,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:18,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:19,765 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:26:20,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:21,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:22,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:24,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:25,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:26:25,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:26:25,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:26:26,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:27,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:28,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:30,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:32,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:32,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:34,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:36,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:38,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:38,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:40,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:26:40,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:26:40,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:26:40,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:42,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:43,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:44,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:46,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:48,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:48,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:49,767 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:26:50,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:52,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:54,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:26:54,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:55,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:26:55,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:26:55,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:26:56,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:58,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:26:59,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:00,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:27:00,503 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:27:00,504 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:27:00,506 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:27:00,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:00,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:27:02,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:04,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:05,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:06,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:08,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:10,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:10,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:27:10,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:27:10,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:27:10,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:12,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:14,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:15,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:16,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:18,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:19,780 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:27:20,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:21,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:22,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:24,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:25,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:27:25,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:27:25,274 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:27:26,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:26,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:28,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:30,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:32,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:32,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:34,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:36,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:37,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:38,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:40,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:27:40,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:27:40,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:27:40,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:42,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:43,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:45,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:45,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:27:45,606 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:27:45,607 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:27:45,608 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:27:46,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:46,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:27:47,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:48,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:49,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:49,772 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:27:51,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:53,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:53,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:27:55,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:55,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:27:55,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:27:55,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:27:57,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:59,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:27:59,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:01,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:03,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:04,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:05,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:07,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:09,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:09,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:10,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:28:10,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:28:10,275 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:28:11,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:13,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:15,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:15,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:17,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:19,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:19,774 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:28:20,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:21,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:23,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:25,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:25,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:28:25,288 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:28:25,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:28:26,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:27,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:29,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:30,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:28:30,616 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:28:30,616 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:28:30,618 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:28:31,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:31,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:28:31,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:32,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:33,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:35,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:36,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:37,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:39,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:40,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:28:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:28:40,282 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:28:41,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:42,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:43,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:45,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:47,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:47,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:49,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:49,776 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:28:51,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:53,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:53,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:55,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:55,269 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:28:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:28:55,282 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:28:57,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:28:58,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:28:59,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:01,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:03,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:03,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:05,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:07,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:09,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:09,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:10,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:29:10,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:29:10,296 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:29:11,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:13,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:14,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:15,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:15,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:29:15,630 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:29:15,630 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:29:15,632 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:29:16,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:16,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:29:17,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:19,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:19,794 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:29:20,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:21,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:23,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:25,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:25,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:29:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:29:25,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:29:26,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:27,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:29,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:31,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:31,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:33,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:35,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:37,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:37,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:39,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:40,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:29:40,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:29:40,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:29:41,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:42,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:43,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:45,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:47,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:47,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:49,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:49,781 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:29:51,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:53,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:53,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:55,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:55,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:29:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:29:55,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:29:57,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:29:58,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:29:59,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:00,511 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:30:00,513 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:30:00,513 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:30:00,515 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:30:01,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:01,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:30:02,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:04,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:05,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:07,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:09,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:09,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:10,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:30:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:30:10,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:30:11,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:13,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:14,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:15,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:17,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:19,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:19,783 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:30:20,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:21,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:23,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:25,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:25,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:30:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:30:25,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:30:25,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:27,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:29,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:31,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:31,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:33,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:35,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:36,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:37,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:39,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:40,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:30:40,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:30:40,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:30:41,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:41,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:43,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:45,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:45,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:30:45,559 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:30:45,560 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:30:45,562 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:30:46,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:46,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:30:47,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:49,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:49,785 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:30:51,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:52,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:53,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:55,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:55,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:30:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:30:55,289 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:30:57,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:30:58,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:30:59,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:01,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:03,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:03,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:05,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:07,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:08,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:09,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:10,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:31:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:31:10,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:31:11,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:13,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:14,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:15,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:17,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:19,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:19,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:19,787 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:31:21,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:23,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:25,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:25,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:25,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:31:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:31:25,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:31:27,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:29,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:30,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:30,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:31:30,535 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:31:30,536 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:31:30,537 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:31:31,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:31,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:31:32,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:35,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:35,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:37,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:39,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:40,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:31:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:31:40,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:31:41,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:41,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:43,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:45,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:46,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:47,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:49,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:49,789 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:31:51,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:52,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:53,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:55,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:31:55,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:31:55,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:31:55,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:57,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:31:57,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:31:59,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:01,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:02,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:03,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:05,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:07,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:08,270 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:09,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:10,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:32:10,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:32:10,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:32:11,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:13,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:13,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:15,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:15,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:32:15,493 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:32:15,493 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:32:15,495 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:32:16,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:16,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:32:19,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:19,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:19,791 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:32:21,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:23,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:24,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:25,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:32:25,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:32:25,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:32:25,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:27,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:29,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:29,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:31,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:33,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:35,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:35,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:37,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:39,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:40,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:32:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:32:40,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:32:40,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:41,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:43,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:45,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:46,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:47,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:49,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:49,794 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:32:51,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:51,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:53,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:55,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:32:55,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:32:55,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:32:55,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:56,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:32:57,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:32:59,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:00,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:33:00,527 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:33:00,527 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:33:00,529 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:33:01,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:01,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:33:02,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:02,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:05,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:07,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:07,688 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:09,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:10,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:33:10,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:33:10,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:33:11,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:13,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:13,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:15,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:17,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:18,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:19,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:19,796 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:33:21,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:23,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:23,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:25,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:33:25,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:33:25,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:33:25,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:27,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:29,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:29,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:31,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:33,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:34,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:35,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:37,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:39,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:40,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:40,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:33:40,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:33:40,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:33:41,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:43,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:45,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:45,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:33:45,376 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:33:45,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:33:45,378 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:33:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:33:46,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:49,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:49,798 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:33:50,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:51,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:53,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:55,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:33:55,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:33:55,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:33:55,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:56,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:33:57,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:33:59,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:01,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:01,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:03,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:05,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:06,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:07,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:09,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:10,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:34:10,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:34:10,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:34:11,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:12,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:13,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:15,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:17,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:17,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:19,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:19,801 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:34:21,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:23,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:23,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:25,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:34:25,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:34:25,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:34:25,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:27,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:28,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:29,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:30,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:34:30,334 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:34:30,335 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:34:30,337 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:34:30,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:34:31,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:32,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:33,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:35,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:37,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:39,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:39,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:40,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:34:40,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:34:40,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:34:41,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:43,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:44,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:45,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:47,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:49,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:49,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:34:49,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:51,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:53,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:55,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:34:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:34:55,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:34:55,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:34:55,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:57,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:34:59,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:00,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:01,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:03,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:05,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:06,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:07,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:09,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:35:10,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:35:10,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:35:11,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:11,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:13,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:15,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:35:15,339 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:35:15,339 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:35:15,340 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:35:15,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:15,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:35:16,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:17,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:19,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:19,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:35:21,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:22,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:23,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:35:25,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:35:25,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:35:25,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:27,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:27,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:29,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:31,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:33,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:33,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:35,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:37,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:38,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:39,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:35:40,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:35:40,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:35:41,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:43,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:44,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:45,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:47,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:49,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:49,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:49,807 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:35:51,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:53,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:54,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:35:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:35:55,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:35:55,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:35:55,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:57,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:35:59,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:00,299 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:00,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:36:00,333 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:36:00,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:36:00,335 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:36:00,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:36:01,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:02,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:05,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:05,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:07,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:09,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:36:10,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:36:10,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:36:11,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:11,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:13,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:15,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:16,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:17,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:19,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:19,809 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:36:21,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:21,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:23,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:36:25,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:36:25,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:36:25,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:27,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:27,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:29,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:31,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:32,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:33,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:35,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:37,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:37,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:39,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:36:40,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:36:40,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:36:41,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:43,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:43,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:45,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:36:45,172 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:36:45,173 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:36:45,176 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:36:45,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:45,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:36:46,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:48,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:49,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:49,815 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:36:51,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:53,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:54,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:36:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:36:55,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:36:55,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:57,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:36:59,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:36:59,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:01,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:03,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:04,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:05,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:07,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:09,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:10,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:37:10,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:37:10,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:37:11,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:13,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:15,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:15,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:17,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:19,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:19,817 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:37:20,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:21,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:23,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:25,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:37:25,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:37:25,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:37:25,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:26,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:27,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:29,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:30,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:37:30,054 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:37:30,054 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:37:30,055 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:37:30,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:37:31,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:31,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:32,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:35,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:37,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:37,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:39,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:40,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:37:40,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:37:40,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:37:41,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:42,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:42,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:44,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:46,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:47,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:48,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:49,819 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:37:50,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:52,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:53,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:54,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:55,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:37:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:37:55,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:37:56,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:37:58,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:37:58,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:00,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:02,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:04,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:04,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:06,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:08,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:09,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:10,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:38:10,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:38:10,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:38:10,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:12,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:14,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:15,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:15,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:38:15,035 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:38:15,036 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:38:15,037 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:38:15,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:38:16,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:18,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:19,822 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:38:20,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:20,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:22,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:24,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:25,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:38:25,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:38:25,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:38:25,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:26,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:28,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:30,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:31,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:33,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:35,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:36,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:37,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:39,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:40,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:38:40,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:38:40,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:38:41,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:41,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:43,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:45,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:47,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:47,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:49,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:49,824 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:38:51,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:52,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:53,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:55,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:55,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:38:55,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:38:55,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:38:57,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:58,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:38:59,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:38:59,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:38:59,824 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:38:59,824 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:38:59,826 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:39:00,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:39:01,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:03,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:03,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:05,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:07,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:08,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:09,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:10,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:39:10,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:39:10,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:39:11,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:13,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:14,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:15,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:17,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:19,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:19,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:19,826 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:39:21,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:23,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:24,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:25,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:25,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:39:25,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:39:25,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:39:27,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:29,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:30,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:31,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:33,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:35,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:35,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:37,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:39,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:40,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:39:40,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:39:40,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:39:41,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:41,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:43,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:44,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:39:44,720 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:39:44,720 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:39:44,722 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:39:45,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:45,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:39:46,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:47,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:49,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:49,829 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:39:51,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:51,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:53,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:55,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:55,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:39:55,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:39:55,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:39:57,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:39:57,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:39:59,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:01,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:02,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:03,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:05,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:07,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:09,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:10,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:40:10,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:40:10,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:40:11,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:13,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:13,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:15,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:17,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:18,859 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:19,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:19,831 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:40:21,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:23,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:24,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:25,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:25,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:40:25,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:40:25,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:40:27,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:29,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:29,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:29,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:40:29,662 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:40:29,662 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:40:29,664 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:40:30,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:30,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:40:33,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:35,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:35,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:37,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:39,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:40,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:40,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:40:40,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:40:40,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:40:41,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:43,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:45,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:45,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:47,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:49,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:49,833 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:40:50,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:51,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:53,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:55,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:55,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:40:55,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:40:55,294 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:40:56,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:40:57,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:40:59,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:01,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:01,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:03,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:05,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:07,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:07,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:09,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:10,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:41:10,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:41:10,294 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:41:11,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:12,653 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:13,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:14,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:41:14,476 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:41:14,476 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:41:14,478 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:41:15,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:15,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:41:16,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:18,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:19,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:19,835 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:41:21,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:23,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:23,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:25,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:25,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:41:25,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:41:25,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:41:27,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:28,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:29,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:31,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:33,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:34,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:35,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:37,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:39,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:39,619 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:40,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:41:40,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:41:40,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:41:41,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:43,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:45,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:45,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:47,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:49,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:49,848 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:41:50,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:51,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:53,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:55,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:55,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:41:55,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:41:55,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:41:55,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:41:57,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:59,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:41:59,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:41:59,393 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:41:59,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:41:59,395 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:42:00,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:00,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:42:01,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:03,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:05,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:06,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:07,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:09,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:10,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:42:10,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:42:10,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:42:11,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:11,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:13,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:15,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:17,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:17,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:19,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:19,839 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:42:21,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:22,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:23,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:25,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:25,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:42:25,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:42:25,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:42:27,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:28,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:29,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:31,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:33,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:33,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:35,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:37,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:38,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:39,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:42:40,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:42:40,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:42:41,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:43,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:44,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:44,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:42:44,277 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:42:44,277 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:42:44,280 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:42:44,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:42:45,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:46,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:49,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:49,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:49,841 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:42:51,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:53,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:55,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:42:55,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:42:55,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:42:55,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:42:55,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:57,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:42:59,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:00,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:01,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:03,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:05,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:05,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:07,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:09,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:10,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:43:10,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:43:10,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:43:11,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:11,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:13,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:15,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:16,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:17,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:19,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:19,881 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:43:21,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:21,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:23,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:25,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:43:25,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:43:25,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:43:25,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:27,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:27,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:29,096 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:43:29,097 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:43:29,098 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:43:29,100 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:43:29,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:29,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:43:30,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:32,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:33,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:35,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:37,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:38,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:39,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:43:40,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:43:40,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:43:41,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:43,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:43,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:45,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:47,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:48,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:49,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:49,893 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:43:51,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:53,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:54,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:43:55,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:43:55,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:43:55,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:43:55,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:57,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:59,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:43:59,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:01,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:03,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:04,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:05,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:07,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:09,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:10,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:44:10,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:10,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:44:10,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:44:11,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:13,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:13,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:44:13,987 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:44:13,987 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:44:13,989 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:44:14,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:44:15,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:15,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:16,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:19,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:19,886 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:44:20,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:21,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:23,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:25,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:44:25,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:44:25,301 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:44:25,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:26,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:27,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:29,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:31,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:31,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:33,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:35,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:37,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:37,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:39,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:40,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:44:40,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:44:40,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:44:41,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:42,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:43,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:45,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:47,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:48,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:49,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:49,888 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:44:51,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:53,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:53,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:44:55,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:44:55,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:44:55,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:57,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:59,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:44:59,041 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:44:59,043 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:44:59,043 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:44:59,045 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:44:59,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:44:59,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:45:00,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:03,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:04,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:05,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:07,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:09,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:09,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:45:10,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:45:10,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:45:11,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:13,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:15,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:15,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:17,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:19,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:19,891 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:45:20,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:21,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:23,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:45:25,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:45:25,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:45:25,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:26,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:27,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:29,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:31,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:31,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:33,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:35,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:36,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:37,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:39,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:40,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:45:40,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:45:40,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:45:41,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:42,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:43,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:44,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:45:44,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:45:44,203 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:45:44,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:45:44,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:45:45,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:46,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:47,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:49,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:49,905 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:45:51,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:53,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:53,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:55,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:45:55,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:45:55,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:45:55,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:57,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:45:58,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:45:59,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:01,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:03,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:04,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:05,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:07,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:09,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:09,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:10,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:46:10,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:46:10,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:46:11,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:13,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:15,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:15,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:17,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:19,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:19,895 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:46:20,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:21,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:23,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:25,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:46:25,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:46:25,302 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:46:25,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:25,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:27,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:29,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:46:29,448 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:46:29,448 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:46:29,450 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:46:29,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:29,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:46:30,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:31,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:31,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:33,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:35,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:36,618 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:37,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:39,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:40,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:46:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:46:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:46:41,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:42,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:43,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:45,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:47,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:47,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:49,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:49,897 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:46:51,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:52,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:53,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:55,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:46:55,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:46:55,323 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:46:55,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:57,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:46:58,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:46:59,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:01,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:03,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:03,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:05,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:07,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:09,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:09,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:10,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:47:10,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:47:10,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:47:11,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:13,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:14,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:14,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:47:14,625 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:47:14,626 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:47:14,629 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:47:15,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:15,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:47:16,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:17,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:19,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:19,901 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:47:19,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:21,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:23,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:25,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:47:25,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:25,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:47:25,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:47:25,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:27,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:29,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:30,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:31,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:33,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:35,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:36,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:37,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:39,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:40,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:47:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:47:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:47:41,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:41,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:43,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:45,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:47,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:47,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:49,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:49,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:47:51,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:52,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:53,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:55,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:47:55,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:47:55,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:47:55,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:57,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:47:57,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:59,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:47:59,720 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:47:59,720 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:47:59,722 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:47:59,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:47:59,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:48:00,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:01,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:03,302 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:03,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:05,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:07,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:08,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:09,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:10,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:48:10,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:48:10,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:48:11,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:13,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:14,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:15,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:17,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:19,466 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:19,906 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:48:19,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:21,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:23,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:24,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:25,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:48:25,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:48:25,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:48:25,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:27,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:29,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:30,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:31,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:33,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:35,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:35,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:37,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:39,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:40,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:48:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:48:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:48:41,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:41,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:43,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:44,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:48:44,706 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:48:44,707 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:48:44,709 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:48:44,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:48:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:46,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:46,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:47,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:49,908 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:48:49,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:51,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:51,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:53,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:55,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:48:55,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:48:55,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:48:55,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:57,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:48:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:48:59,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:01,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:02,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:03,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:05,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:07,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:08,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:09,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:10,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:49:10,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:49:10,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:49:11,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:13,519 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:13,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:15,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:17,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:18,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:19,910 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:49:19,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:22,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:24,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:24,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:25,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:49:25,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:49:25,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:49:26,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:28,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:29,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:29,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:49:29,751 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:49:29,751 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:49:29,753 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:49:30,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:30,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:49:31,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:32,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:34,015 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:35,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:36,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:38,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:40,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:40,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:49:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:40,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:49:40,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:49:42,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:44,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:45,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:46,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:48,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:49,929 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:49:50,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:50,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:52,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:54,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:55,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:49:55,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:49:55,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:49:56,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:49:56,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:49:58,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:00,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:02,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:02,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:04,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:06,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:07,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:08,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:10,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:10,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:50:10,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:50:10,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:50:12,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:12,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:14,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:14,524 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:50:14,526 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:50:14,526 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:50:14,528 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:50:15,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:50:16,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:17,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:18,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:19,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:50:20,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:22,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:23,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:24,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:25,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:50:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:50:25,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:50:26,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:28,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:28,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:30,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:32,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:34,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:34,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:36,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:38,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:39,432 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:40,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:40,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:50:40,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:50:40,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:50:42,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:44,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:44,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:45,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:47,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:49,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:49,917 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:50:49,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:51,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:53,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:55,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:55,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:50:55,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:50:55,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:50:55,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:50:57,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:59,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:50:59,047 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:50:59,048 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:50:59,049 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:50:59,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:50:59,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:51:00,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:01,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:03,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:05,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:06,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:07,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:09,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:10,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:51:10,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:51:10,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:51:11,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:11,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:13,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:15,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:16,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:17,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:19,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:19,919 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:51:21,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:22,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:23,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:25,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:25,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:51:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:51:25,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:51:27,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:27,503 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:29,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:31,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:32,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:33,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:35,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:37,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:38,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:39,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:40,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:51:40,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:51:40,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:51:41,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:43,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:43,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:43,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:51:43,559 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:51:43,559 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:51:43,560 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:51:44,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:51:45,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:47,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:48,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:49,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:49,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:51:51,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:53,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:54,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:51:55,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:55,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:51:55,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:51:55,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:51:57,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:59,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:51:59,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:01,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:03,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:04,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:05,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:07,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:09,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:10,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:10,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:52:10,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:52:10,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:52:11,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:13,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:15,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:15,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:17,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:19,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:19,924 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:52:20,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:21,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:23,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:25,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:25,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:52:25,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:52:25,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:52:26,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:27,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:28,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:52:28,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:52:28,024 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:52:28,026 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:52:28,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:52:29,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:30,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:31,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:31,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:33,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:35,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:36,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:37,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:39,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:40,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:52:40,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:52:40,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:52:41,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:42,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:43,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:45,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:47,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:47,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:49,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:49,926 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:52:51,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:52,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:53,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:55,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:55,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:52:55,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:52:55,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:52:57,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:52:58,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:52:59,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:01,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:03,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:03,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:05,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:07,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:09,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:09,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:10,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:53:10,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:53:10,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:53:11,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:12,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:53:12,627 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:53:12,627 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:53:12,629 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:53:13,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:13,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:53:14,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:14,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:15,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:17,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:19,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:19,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:19,928 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:53:21,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:23,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:25,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:25,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:25,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:53:25,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:53:25,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:53:27,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:29,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:30,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:31,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:33,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:35,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:35,770 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:37,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:39,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:40,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:53:40,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:53:40,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:53:41,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:41,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:43,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:45,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:46,470 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:47,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:49,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:49,931 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:53:51,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:51,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:53,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:55,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:55,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:53:55,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:53:55,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:53:57,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:53:57,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:53:57,194 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:53:57,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:53:57,196 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:53:57,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:57,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:53:58,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:53:59,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:01,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:02,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:03,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:05,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:07,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:07,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:09,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:10,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:54:10,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:54:10,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:54:11,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:13,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:13,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:15,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:17,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:18,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:19,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:19,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:54:21,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:23,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:23,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:25,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:54:25,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:25,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:54:25,326 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:54:27,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:29,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:29,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:31,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:33,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:34,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:35,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:37,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:39,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:40,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:54:40,326 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:54:40,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:54:41,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:41,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:54:41,664 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:54:41,664 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:54:41,666 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:54:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:42,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:54:43,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:45,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:45,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:47,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:49,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:49,934 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:54:50,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:51,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:53,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:55,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:54:55,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:54:55,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:54:55,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:56,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:54:57,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:54:59,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:01,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:02,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:03,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:05,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:07,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:07,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:09,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:10,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:55:10,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:55:10,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:55:11,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:12,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:13,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:15,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:17,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:18,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:19,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:19,937 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:55:21,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:23,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:23,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:25,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:55:25,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:55:25,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:55:25,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:26,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:55:26,299 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:55:26,300 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:55:26,301 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:55:26,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:55:27,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:28,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:29,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:29,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:31,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:33,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:34,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:35,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:37,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:39,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:39,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:40,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:55:40,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:55:40,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:55:41,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:43,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:45,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:45,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:47,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:49,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:49,939 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:55:50,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:51,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:53,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:55,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:55:55,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:55:55,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:55:55,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:56,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:55:57,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:55:59,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:01,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:01,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:03,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:05,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:07,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:07,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:09,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:10,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:56:10,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:56:10,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:56:10,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:56:10,925 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:56:10,926 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:56:10,928 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:56:11,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:11,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:56:12,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:12,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:13,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:15,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:17,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:18,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:19,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:19,941 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:56:21,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:23,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:23,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:25,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:56:25,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:56:25,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:56:25,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:27,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:28,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:29,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:31,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:33,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:34,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:35,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:37,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:39,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:39,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:40,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:56:40,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:56:40,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:56:41,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:43,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:44,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:45,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:47,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:49,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:49,943 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:56:49,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:51,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:53,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:55,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:56:55,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:56:55,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:56:55,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:56:55,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:55,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:56:55,529 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:56:55,530 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:56:55,533 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:56:56,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:56,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:56:57,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:56:59,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:00,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:01,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:03,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:05,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:06,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:07,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:09,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:10,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:57:10,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:57:10,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:57:11,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:11,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:13,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:15,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:16,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:17,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:19,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:19,945 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:57:21,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:22,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:23,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:25,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:57:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:57:25,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:57:25,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:27,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:27,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:29,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:31,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:32,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:33,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:35,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:37,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:38,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:39,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:40,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:57:40,149 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:57:40,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:57:40,152 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:57:40,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:57:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:57:40,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:57:40,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:57:41,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:42,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:43,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:43,687 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:45,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:47,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:49,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:49,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:49,948 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:57:51,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:53,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:54,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:57:55,307 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:57:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:57:55,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:57:55,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:57,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:59,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:57:59,839 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:01,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:03,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:05,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:05,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:07,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:09,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:10,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:58:10,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:58:10,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:58:11,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:13,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:15,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:15,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:17,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:19,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:19,968 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:58:20,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:21,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:23,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:24,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:58:24,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:58:24,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:58:24,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:58:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:58:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:58:25,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:58:25,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:25,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:58:26,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:27,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:29,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:31,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:33,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:33,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:35,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:37,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:39,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:39,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:40,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:58:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:58:40,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:58:41,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:43,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:44,368 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:45,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:47,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:49,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:49,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:49,954 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:58:51,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:53,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:55,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:58:55,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:58:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:58:55,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:58:55,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:57,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:58:59,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:00,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:01,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:03,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:05,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:05,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:07,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:09,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:59:09,350 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:59:09,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:59:09,352 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:59:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:09,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:59:10,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:59:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:59:10,328 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:59:10,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:11,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:13,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:16,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:19,954 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:59:21,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:21,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:23,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:59:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:59:25,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:59:25,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:27,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:27,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:29,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:31,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:32,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:33,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:35,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:37,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:37,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:39,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:40,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:59:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:59:40,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:59:41,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:43,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:43,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:45,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:47,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:48,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:49,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:49,957 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 17:59:51,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:53,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:53,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:53,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 17:59:53,802 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 17:59:53,803 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 17:59:53,804 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 17:59:54,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:54,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 17:59:55,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 17:59:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 17:59:55,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 17:59:55,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:57,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 17:59:59,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 17:59:59,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:01,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:03,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:04,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:05,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:07,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:09,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:09,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:10,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:00:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:00:10,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:00:11,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:13,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:15,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:15,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:17,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:19,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:19,969 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:00:20,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:21,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:23,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:25,308 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:00:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:00:25,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:00:25,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:25,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:27,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:29,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:31,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:31,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:33,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:35,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:36,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:37,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:38,410 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:00:38,411 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:00:38,411 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:00:38,413 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:00:38,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:00:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:40,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:00:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:00:40,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:00:40,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:41,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:41,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:43,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:45,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:47,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:47,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:49,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:49,961 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:00:51,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:52,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:53,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:00:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:00:55,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:00:55,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:57,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:00:58,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:00:59,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:01,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:03,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:03,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:05,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:07,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:08,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:09,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:10,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:01:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:01:10,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:01:11,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:13,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:14,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:15,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:17,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:19,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:19,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:19,963 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:01:21,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:23,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:01:23,245 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:01:23,246 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:01:23,247 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:01:23,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:23,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:01:24,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:25,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:25,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:01:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:01:25,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:01:25,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:27,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:29,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:30,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:31,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:33,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:35,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:35,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:37,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:39,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:40,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:01:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:01:40,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:01:41,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:41,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:43,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:45,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:46,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:47,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:49,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:49,965 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:01:51,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:51,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:53,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:01:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:01:55,329 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:01:55,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:56,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:01:57,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:01:59,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:01,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:02,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:03,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:05,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:07,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:07,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:02:07,663 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:02:07,663 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:02:07,665 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:02:07,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:07,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:02:08,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:09,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:10,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:02:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:02:10,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:02:11,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:12,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:13,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:15,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:17,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:18,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:19,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:19,981 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:02:21,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:23,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:23,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:25,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:02:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:02:25,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:02:25,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:27,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:28,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:29,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:31,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:33,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:34,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:35,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:37,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:39,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:39,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:02:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:02:40,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:02:41,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:43,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:44,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:45,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:47,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:49,969 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:02:49,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:50,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:51,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:52,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:02:52,114 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:02:52,114 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:02:52,116 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:02:52,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:02:53,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:54,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:02:55,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:02:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:02:55,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:02:55,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:57,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:02:59,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:00,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:01,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:03,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:05,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:06,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:07,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:09,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:10,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:03:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:03:10,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:03:11,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:12,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:14,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:16,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:16,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:18,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:19,971 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:03:20,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:22,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:22,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:24,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:25,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:03:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:03:25,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:03:26,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:27,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:28,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:30,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:32,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:33,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:34,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:36,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:36,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:03:36,672 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:03:36,672 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:03:36,674 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:03:37,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:03:38,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:38,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:39,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:40,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:03:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:03:40,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:03:42,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:43,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:44,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:46,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:48,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:49,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:49,974 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:03:50,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:52,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:54,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:54,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:03:55,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:03:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:03:55,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:03:56,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:58,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:03:59,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:00,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:02,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:04,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:05,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:06,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:08,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:10,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:10,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:04:10,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:04:10,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:04:12,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:14,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:15,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:16,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:18,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:19,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:04:20,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:20,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:21,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:04:21,108 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:04:21,108 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:04:21,109 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:04:22,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:22,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:04:23,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:24,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:25,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:04:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:04:25,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:04:26,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:26,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:28,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:30,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:31,739 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:32,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:34,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:36,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:37,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:38,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:40,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:40,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:04:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:04:40,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:04:42,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:42,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:44,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:46,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:47,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:48,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:49,978 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:04:50,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:52,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:53,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:04:54,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:55,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:04:55,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:04:55,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:04:56,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:58,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:04:58,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:00,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:02,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:03,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:04,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:05,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:05:05,704 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:05:05,704 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:05:05,706 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:05:06,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:06,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:05:07,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:08,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:09,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:10,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:10,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:05:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:05:10,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:05:12,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:14,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:14,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:16,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:18,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:19,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:19,980 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:05:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:22,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:24,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:25,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:05:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:05:25,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:05:26,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:28,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:30,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:30,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:32,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:34,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:35,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:36,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:38,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:40,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:40,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:05:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:05:40,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:05:41,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:42,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:44,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:46,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:46,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:48,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:49,983 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:05:50,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:50,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:05:50,306 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:05:50,306 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:05:50,308 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:05:51,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:05:52,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:52,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:53,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:54,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:55,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:05:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:05:55,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:05:56,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:05:57,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:05:58,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:00,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:02,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:02,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:04,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:06,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:08,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:10,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:10,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:06:10,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:06:10,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:06:12,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:13,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:14,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:16,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:18,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:18,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:19,994 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:06:20,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:22,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:24,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:24,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:25,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:06:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:06:25,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:06:26,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:28,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:29,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:30,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:32,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:34,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:34,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:34,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:06:34,845 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:06:34,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:06:34,847 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:06:35,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:06:36,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:37,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:38,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:40,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:40,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:40,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:06:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:06:40,332 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:06:42,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:44,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:45,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:46,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:48,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:49,986 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:06:50,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:50,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:52,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:54,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:55,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:06:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:06:55,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:06:56,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:06:56,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:06:58,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:00,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:01,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:02,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:04,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:06,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:06,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:08,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:10,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:10,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:07:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:07:10,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:07:12,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:12,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:14,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:16,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:17,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:18,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:19,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:07:19,464 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:07:19,464 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:07:19,466 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:07:19,989 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:07:20,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:20,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:07:21,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:22,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:22,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:24,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:25,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:07:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:07:25,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:07:26,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:28,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:28,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:30,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:32,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:33,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:34,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:36,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:38,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:38,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:40,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:07:40,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:07:40,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:07:42,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:44,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:44,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:46,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:48,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:49,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:49,991 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:07:50,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:52,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:54,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:55,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:07:55,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:07:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:07:55,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:07:56,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:07:58,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:00,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:00,555 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:02,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:04,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:08:04,162 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:08:04,162 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:08:04,165 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:08:04,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:04,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:08:05,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:05,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:06,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:07,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:09,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:10,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:08:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:08:10,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:08:11,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:11,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:13,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:15,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:16,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:17,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:19,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:20,004 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:08:21,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:21,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:23,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:25,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:08:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:08:25,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:08:25,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:27,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:27,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:29,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:31,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:32,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:33,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:35,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:37,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:37,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:39,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:40,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:08:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:08:40,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:08:41,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:43,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:43,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:45,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:47,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:48,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:48,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:08:48,626 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:08:48,626 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:08:48,628 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:08:49,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:49,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:08:49,995 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:08:50,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:51,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:53,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:53,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:55,323 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:08:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:08:55,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:08:55,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:57,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:08:59,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:08:59,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:01,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:03,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:04,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:05,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:07,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:09,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:09,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:10,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:09:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:09:10,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:09:11,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:13,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:15,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:15,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:17,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:19,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:19,997 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:09:20,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:21,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:23,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:25,320 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:09:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:09:25,334 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:09:25,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:26,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:27,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:29,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:31,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:31,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:33,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:09:33,183 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:09:33,183 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:09:33,185 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:09:33,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:33,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:09:34,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:35,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:36,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:37,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:39,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:09:40,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:09:40,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:09:41,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:42,042 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:43,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:45,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:47,380 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:47,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:49,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:49,999 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:09:51,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:52,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:53,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:09:55,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:09:55,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:09:55,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:57,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:09:58,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:09:59,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:01,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:03,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:03,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:05,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:07,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:08,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:09,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:10:10,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:10:10,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:10:11,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:13,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:14,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:15,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:17,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:17,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:10:17,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:10:17,693 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:10:17,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:10:18,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:18,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:10:19,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:19,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:20,013 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:10:21,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:23,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:24,789 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:10:25,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:10:25,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:10:25,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:27,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:29,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:30,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:31,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:33,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:35,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:35,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:37,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:39,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:10:40,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:10:40,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:10:40,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:41,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:43,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:45,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:46,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:47,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:49,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:50,004 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:10:51,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:51,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:53,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:55,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:10:55,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:10:55,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:10:55,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:56,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:10:57,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:10:59,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:01,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:02,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:02,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:11:02,176 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:11:02,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:11:02,178 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:11:02,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:11:03,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:04,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:05,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:07,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:07,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:09,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:10,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:11:10,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:11:10,340 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:11:11,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:12,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:13,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:15,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:17,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:18,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:19,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:20,006 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:11:21,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:23,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:23,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:25,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:11:25,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:11:25,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:11:25,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:27,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:28,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:29,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:31,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:33,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:34,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:35,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:37,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:39,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:39,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:40,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:11:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:11:40,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:11:41,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:43,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:44,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:45,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:46,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:11:46,751 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:11:46,752 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:11:46,753 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:11:47,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:47,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:11:48,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:49,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:50,008 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:11:50,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:51,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:53,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:11:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:11:55,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:11:55,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:11:55,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:57,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:11:59,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:00,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:01,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:03,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:05,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:06,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:07,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:09,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:12:10,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:12:10,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:12:11,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:11,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:13,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:15,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:16,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:17,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:19,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:20,021 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:12:21,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:22,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:23,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:12:25,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:12:25,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:12:25,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:27,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:27,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:29,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:31,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:12:31,270 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:12:31,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:12:31,272 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:12:31,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:31,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:12:32,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:33,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:33,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:35,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:37,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:38,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:39,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:12:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:12:40,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:12:41,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:43,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:43,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:45,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:47,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:49,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:49,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:50,012 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:12:51,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:53,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:54,368 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:12:55,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:12:55,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:12:55,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:57,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:12:59,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:12:59,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:01,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:03,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:05,085 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:05,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:07,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:09,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:13:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:10,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:13:10,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:13:11,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:13,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:15,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:15,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:15,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:13:15,845 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:13:15,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:13:15,847 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:13:16,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:16,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:13:17,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:19,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:20,014 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:13:21,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:21,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:23,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:13:25,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:13:25,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:13:25,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:26,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:27,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:29,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:31,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:31,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:33,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:35,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:37,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:37,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:39,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:40,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:13:40,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:13:40,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:13:41,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:42,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:43,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:45,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:47,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:47,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:49,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:50,017 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:13:51,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:53,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:53,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:55,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:13:55,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:13:55,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:13:55,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:57,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:13:58,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:13:59,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:00,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:14:00,536 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:14:00,536 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:14:00,538 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:14:00,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:14:01,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:02,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:03,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:04,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:05,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:07,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:09,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:09,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:10,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:14:10,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:14:10,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:14:11,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:13,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:14,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:15,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:17,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:19,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:20,030 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:14:20,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:21,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:23,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:25,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:14:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:25,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:14:25,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:14:25,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:27,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:29,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:30,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:31,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:33,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:35,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:36,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:37,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:39,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:14:40,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:14:40,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:14:41,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:41,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:43,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:45,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:14:45,084 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:14:45,084 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:14:45,086 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:14:45,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:45,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:14:46,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:46,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:47,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:49,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:50,021 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:14:51,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:52,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:53,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:14:55,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:14:55,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:14:55,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:57,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:14:57,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:14:59,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:01,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:02,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:03,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:05,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:07,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:08,181 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:09,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:15:10,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:15:10,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:15:11,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:13,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:13,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:15,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:17,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:18,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:19,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:20,023 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:15:21,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:23,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:24,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:15:25,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:15:25,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:15:25,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:27,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:29,625 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:29,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:15:29,658 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:15:29,659 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:15:29,660 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:15:29,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:29,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:15:30,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:31,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:33,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:34,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:35,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:37,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:39,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:40,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:15:40,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:15:40,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:15:41,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:43,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:45,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:45,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:47,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:49,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:50,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:15:50,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:51,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:53,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:15:55,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:15:55,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:15:55,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:56,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:15:57,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:15:59,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:01,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:01,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:03,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:05,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:07,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:07,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:09,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:10,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:16:10,342 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:16:10,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:16:11,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:12,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:13,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:14,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:16:14,314 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:16:14,315 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:16:14,317 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:16:14,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:16:15,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:16,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:17,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:17,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:19,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:20,041 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:16:21,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:23,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:23,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:25,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:16:25,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:16:25,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:16:25,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:27,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:28,570 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:29,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:31,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:33,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:33,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:35,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:37,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:39,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:39,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:40,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:16:40,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:16:40,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:16:41,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:43,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:44,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:45,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:47,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:49,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:49,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:50,030 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:16:51,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:53,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:55,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:16:55,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:16:55,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:16:55,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:16:55,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:57,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:16:58,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:16:58,824 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:16:58,824 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:16:58,826 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:16:58,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:16:59,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:00,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:00,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:01,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:03,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:05,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:05,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:07,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:09,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:17:10,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:17:10,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:17:11,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:11,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:13,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:15,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:16,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:17,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:19,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:20,032 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:17:21,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:22,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:23,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:17:25,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:17:25,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:17:26,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:27,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:28,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:30,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:32,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:32,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:34,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:36,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:38,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:40,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:40,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:17:40,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:17:40,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:17:42,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:43,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:43,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:17:43,519 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:17:43,520 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:17:43,521 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:17:44,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:44,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:17:45,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:46,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:48,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:48,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:50,034 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:17:50,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:52,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:54,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:54,183 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:17:55,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:17:55,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:17:55,343 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:17:56,176 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:58,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:17:59,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:00,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:02,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:04,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:04,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:06,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:08,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:10,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:10,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:18:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:18:10,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:18:12,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:14,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:15,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:16,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:18,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:20,047 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:18:20,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:20,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:22,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:24,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:18:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:18:25,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:18:26,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:26,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:28,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:18:28,144 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:18:28,145 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:18:28,148 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:18:28,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:28,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:18:29,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:30,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:31,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:32,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:34,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:36,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:37,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:38,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:40,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:40,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:18:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:18:40,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:18:42,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:42,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:44,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:46,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:47,699 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:48,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:50,039 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:18:50,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:52,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:53,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:18:54,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:55,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:18:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:18:55,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:18:56,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:58,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:18:58,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:00,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:02,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:03,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:04,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:06,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:08,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:09,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:10,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:10,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:19:10,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:19:10,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:19:12,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:12,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:19:12,757 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:19:12,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:19:12,759 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:19:13,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:19:14,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:14,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:15,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:16,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:18,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:19,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:20,042 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:19:20,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:22,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:24,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:25,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:25,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:19:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:19:25,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:19:26,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:28,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:30,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:30,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:32,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:34,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:35,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:36,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:38,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:19:40,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:19:40,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:19:41,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:42,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:44,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:46,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:46,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:48,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:50,045 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:19:50,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:51,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:52,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:54,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:19:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:19:55,372 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:19:56,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:57,338 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:19:57,369 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:19:57,371 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:19:57,371 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:19:57,373 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:19:57,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:19:58,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:19:59,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:00,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:02,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:02,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:04,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:06,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:08,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:08,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:10,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:20:10,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:20:10,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:20:10,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:12,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:13,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:14,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:16,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:18,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:18,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:20,047 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:20:20,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:22,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:24,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:24,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:20:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:20:25,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:20:26,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:28,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:29,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:30,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:32,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:34,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:34,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:36,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:38,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:40,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:20:40,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:20:40,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:20:40,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:41,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:20:41,943 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:20:41,944 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:20:41,946 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:20:42,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:42,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:20:43,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:44,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:45,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:46,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:48,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:50,049 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:20:50,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:50,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:52,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:54,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:20:55,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:20:55,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:20:56,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:20:56,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:20:58,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:00,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:01,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:02,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:04,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:06,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:06,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:08,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:10,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:21:10,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:21:10,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:21:10,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:12,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:12,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:14,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:16,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:17,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:18,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:20,051 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:21:20,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:22,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:22,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:24,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:25,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:21:25,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:21:25,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:21:26,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:26,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:21:26,616 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:21:26,616 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:21:26,618 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:21:27,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:21:28,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:28,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:30,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:32,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:33,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:34,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:36,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:38,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:39,056 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:40,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:21:40,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:21:40,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:21:40,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:42,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:44,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:44,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:46,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:48,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:49,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:50,053 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:21:50,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:52,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:54,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:55,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:21:55,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:21:55,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:21:55,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:21:56,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:21:58,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:00,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:00,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:02,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:04,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:05,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:06,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:08,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:22:10,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:22:10,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:22:10,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:11,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:11,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:22:11,342 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:22:11,342 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:22:11,344 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:22:11,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:22:12,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:14,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:16,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:16,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:18,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:20,055 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:22:20,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:21,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:22,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:24,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:22:25,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:22:25,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:22:26,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:27,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:28,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:30,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:32,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:32,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:34,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:36,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:38,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:38,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:22:40,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:22:40,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:22:40,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:42,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:43,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:44,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:46,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:48,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:48,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:50,057 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:22:50,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:52,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:54,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:22:54,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:22:55,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:22:55,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:22:55,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:22:55,881 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:22:55,882 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:22:55,884 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:22:56,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:56,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:22:58,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:22:59,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:00,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:02,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:04,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:04,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:06,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:08,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:10,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:10,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:23:10,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:23:10,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:23:10,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:12,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:14,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:15,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:16,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:18,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:20,059 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:23:20,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:20,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:22,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:24,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:23:25,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:23:25,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:23:26,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:26,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:28,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:30,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:31,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:32,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:34,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:36,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:36,892 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:38,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:40,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:23:40,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:23:40,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:23:40,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:23:40,489 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:23:40,490 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:23:40,492 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:23:40,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:40,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:23:42,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:42,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:44,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:46,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:47,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:48,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:50,061 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:23:50,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:52,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:52,974 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:54,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:55,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:23:55,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:23:55,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:23:56,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:23:58,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:23:58,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:00,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:02,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:03,678 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:04,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:06,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:08,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:09,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:10,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:24:10,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:24:10,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:24:10,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:12,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:14,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:14,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:16,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:18,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:19,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:20,063 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:24:20,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:22,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:24,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:25,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:25,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:24:25,124 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:24:25,125 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:24:25,127 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:24:25,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:24:25,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:24:25,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:24:25,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:24:26,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:28,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:30,461 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:30,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:32,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:34,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:35,817 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:36,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:38,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:40,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:24:40,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:24:40,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:24:40,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:41,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:42,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:44,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:46,514 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:46,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:48,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:50,066 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:24:50,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:51,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:52,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:54,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:55,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:24:55,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:24:55,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:24:56,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:24:57,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:24:58,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:00,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:02,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:02,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:04,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:06,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:07,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:08,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:09,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:25:09,743 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:25:09,743 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:25:09,745 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:25:10,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:25:10,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:25:10,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:25:10,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:10,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:25:12,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:13,313 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:14,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:16,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:18,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:18,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:20,068 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:25:20,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:22,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:24,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:24,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:25,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:25:25,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:25:25,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:25:26,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:28,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:29,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:29,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:31,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:33,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:34,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:35,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:37,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:39,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:40,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:40,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:25:40,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:25:40,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:25:41,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:43,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:45,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:45,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:47,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:49,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:50,070 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:25:50,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:51,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:53,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:54,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:25:54,426 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:25:54,427 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:25:54,429 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:25:54,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:25:55,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:25:55,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:25:55,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:25:55,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:56,192 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:25:56,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:57,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:25:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:01,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:03,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:05,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:06,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:07,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:09,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:10,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:26:10,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:26:10,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:26:11,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:12,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:13,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:15,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:17,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:17,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:19,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:20,073 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:26:21,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:22,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:23,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:25,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:26:25,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:26:25,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:26:25,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:27,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:28,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:29,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:31,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:33,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:33,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:35,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:37,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:39,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:39,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:26:39,061 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:26:39,062 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:26:39,063 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:26:39,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:39,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:26:40,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:26:40,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:26:40,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:26:40,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:41,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:43,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:44,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:45,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:47,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:49,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:49,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:50,075 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:26:51,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:53,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:55,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:26:55,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:26:55,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:26:55,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:26:55,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:57,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:26:59,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:00,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:01,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:03,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:05,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:05,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:07,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:09,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:10,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:27:10,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:27:10,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:27:11,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:11,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:13,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:15,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:16,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:17,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:19,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:20,077 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:27:21,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:21,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:23,741 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:27:23,742 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:27:23,743 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:27:23,745 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:27:23,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:23,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:27:24,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:25,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:27:25,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:27:25,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:27:25,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:27,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:27,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:29,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:31,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:32,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:33,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:35,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:37,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:37,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:39,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:40,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:27:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:27:40,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:27:41,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:43,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:43,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:45,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:47,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:48,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:49,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:50,080 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:27:51,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:53,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:54,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:27:55,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:27:55,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:27:55,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:27:55,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:58,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:27:59,341 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:00,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:02,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:04,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:04,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:06,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:08,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:08,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:28:08,258 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:28:08,258 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:28:08,260 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:28:09,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:09,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:28:10,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:10,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:10,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:28:10,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:28:10,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:28:12,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:14,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:15,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:16,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:18,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:20,081 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:28:20,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:20,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:22,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:24,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:25,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:28:25,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:28:25,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:28:26,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:26,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:28,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:30,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:31,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:32,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:34,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:36,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:36,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:38,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:40,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:40,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:28:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:28:40,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:28:42,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:42,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:44,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:46,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:47,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:48,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:50,094 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:28:50,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:52,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:52,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:28:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:28:53,035 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:28:53,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:28:53,038 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:28:53,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:28:54,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:55,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:55,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:28:55,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:28:55,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:28:56,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:58,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:28:58,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:00,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:02,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:03,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:04,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:06,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:08,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:09,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:10,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:10,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:29:10,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:29:10,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:29:12,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:14,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:14,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:16,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:18,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:19,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:20,086 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:29:20,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:22,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:24,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:25,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:25,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:29:25,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:29:25,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:29:26,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:28,233 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:30,235 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:30,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:32,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:34,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:35,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:36,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:37,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:29:37,566 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:29:37,567 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:29:37,568 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:29:38,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:38,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:29:39,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:40,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:40,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:29:40,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:29:40,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:29:41,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:42,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:44,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:46,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:46,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:48,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:50,088 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:29:50,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:51,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:52,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:54,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:55,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:29:55,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:29:55,409 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:29:56,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:29:57,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:29:58,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:00,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:02,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:02,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:04,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:06,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:07,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:08,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:10,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:10,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:30:10,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:30:10,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:30:12,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:13,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:14,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:16,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:18,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:18,539 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:20,090 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:30:20,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:22,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:30:22,134 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:30:22,135 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:30:22,136 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:30:22,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:22,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:30:23,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:23,891 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:24,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:25,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:30:25,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:30:25,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:30:26,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:28,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:29,239 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:30,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:32,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:34,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:34,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:36,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:38,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:39,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:40,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:40,336 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:30:40,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:30:40,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:30:42,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:44,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:45,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:46,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:48,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:50,104 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:30:50,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:50,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:52,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:54,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:55,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:30:55,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:30:55,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:30:56,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:30:56,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:30:58,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:00,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:01,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:02,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:04,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:06,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:06,746 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:06,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:31:06,779 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:31:06,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:31:06,781 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:31:07,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:31:08,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:09,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:10,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:10,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:31:10,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:31:10,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:31:12,095 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:12,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:14,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:16,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:17,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:18,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:20,095 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:31:20,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:22,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:22,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:24,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:25,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:31:25,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:31:25,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:31:26,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:28,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:28,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:30,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:32,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:33,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:34,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:36,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:38,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:38,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:40,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:31:40,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:40,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:31:40,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:31:42,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:44,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:44,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:46,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:48,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:49,656 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:50,096 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:31:50,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:51,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:31:51,472 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:31:51,473 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:31:51,474 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:31:52,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:52,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:31:53,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:54,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:55,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:31:55,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:31:55,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:31:55,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:31:56,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:31:58,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:00,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:00,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:02,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:04,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:05,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:06,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:08,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:10,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:32:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:32:10,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:32:10,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:11,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:12,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:14,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:16,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:16,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:18,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:20,099 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:32:20,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:21,706 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:22,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:24,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:25,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:32:25,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:32:25,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:32:26,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:27,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:28,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:30,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:32,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:32,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:34,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:36,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:32:36,001 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:32:36,002 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:32:36,004 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:32:36,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:36,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:32:37,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:37,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:38,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:40,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:32:40,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:32:40,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:32:40,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:42,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:43,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:44,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:46,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:48,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:48,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:50,111 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:32:50,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:52,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:53,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:32:54,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:55,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:32:55,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:32:55,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:32:56,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:58,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:32:59,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:00,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:02,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:04,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:04,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:06,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:08,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:09,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:10,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:33:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:33:10,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:33:10,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:12,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:14,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:15,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:16,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:18,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:20,103 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:33:20,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:20,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:20,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:33:20,671 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:33:20,671 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:33:20,673 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:33:21,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:33:22,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:23,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:24,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:25,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:33:25,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:33:25,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:33:25,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:26,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:28,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:30,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:31,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:32,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:34,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:36,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:36,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:38,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:40,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:33:40,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:33:40,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:33:40,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:42,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:42,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:44,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:46,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:47,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:48,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:50,105 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:33:50,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:52,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:52,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:54,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:55,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:33:55,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:33:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:33:56,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:33:58,086 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:33:58,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:00,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:02,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:03,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:04,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:05,230 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:34:05,231 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:34:05,232 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:34:05,234 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:34:05,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:34:06,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:07,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:08,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:08,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:10,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:34:10,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:34:10,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:34:10,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:12,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:14,138 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:14,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:16,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:18,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:19,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:20,107 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:34:20,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:22,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:24,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:24,909 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:25,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:34:25,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:34:25,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:34:26,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:28,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:30,280 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:30,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:32,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:34,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:35,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:36,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:38,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:40,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:34:40,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:34:40,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:34:40,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:40,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:42,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:44,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:46,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:46,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:48,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:49,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:34:49,904 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:34:49,904 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:34:49,906 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:34:50,129 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:34:50,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:50,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:34:51,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:52,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:54,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:55,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:34:55,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:34:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:34:56,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:34:56,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:34:58,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:00,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:02,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:02,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:04,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:06,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:07,687 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:08,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:10,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:35:10,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:35:10,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:35:10,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:12,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:13,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:14,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:16,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:18,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:18,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:20,112 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:35:20,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:22,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:23,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:24,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:25,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:35:25,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:35:25,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:35:26,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:28,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:29,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:30,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:32,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:34,422 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:34,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:35:34,455 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:35:34,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:35:34,458 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:35:34,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:34,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:35:36,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:38,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:39,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:35:40,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:35:40,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:35:40,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:42,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:44,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:45,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:46,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:48,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:50,114 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:35:50,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:50,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:52,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:54,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:55,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:35:55,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:35:55,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:35:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:35:56,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:35:58,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:00,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:01,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:02,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:04,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:06,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:06,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:08,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:10,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:36:10,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:36:10,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:36:10,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:11,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:12,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:14,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:16,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:17,197 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:18,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:19,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:36:19,006 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:36:19,006 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:36:19,008 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:36:19,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:36:20,116 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:36:20,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:22,537 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:22,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:24,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:25,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:36:25,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:36:25,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:36:26,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:27,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:28,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:30,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:32,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:33,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:34,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:36,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:38,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:38,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:40,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:36:40,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:36:40,407 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:36:40,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:42,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:43,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:44,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:46,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:48,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:49,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:50,131 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:36:50,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:52,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:54,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:36:54,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:55,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:36:55,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:36:55,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:36:56,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:58,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:36:59,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:00,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:02,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:03,579 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:37:03,581 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:37:03,581 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:37:03,583 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:37:03,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:37:04,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:05,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:06,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:08,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:10,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:37:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:10,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:37:10,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:37:10,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:12,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:14,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:16,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:16,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:18,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:20,121 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:37:20,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:21,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:22,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:24,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:25,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:37:25,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:37:25,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:37:26,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:26,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:28,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:30,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:32,171 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:32,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:34,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:36,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:37,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:38,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:40,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:37:40,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:37:40,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:37:40,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:42,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:42,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:44,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:46,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:48,158 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:48,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:37:48,197 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:37:48,198 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:37:48,200 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:37:48,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:48,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:37:50,123 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:37:50,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:52,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:53,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:54,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:55,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:37:55,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:37:55,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:37:56,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:58,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:37:58,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:37:59,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:01,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:03,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:04,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:05,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:08,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:09,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:10,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:10,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:38:10,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:38:10,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:38:12,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:14,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:14,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:16,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:18,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:20,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:20,125 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:38:20,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:22,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:24,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:25,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:38:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:25,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:38:25,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:38:26,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:28,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:30,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:30,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:32,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:32,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:38:32,811 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:38:32,811 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:38:32,813 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:38:33,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:38:34,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:35,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:36,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:36,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:38,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:40,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:40,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:38:40,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:38:40,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:38:41,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:42,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:44,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:46,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:47,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:48,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:50,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:50,138 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:38:52,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:52,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:54,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:55,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:38:55,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:38:55,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:38:56,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:38:57,748 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:38:58,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:00,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:02,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:03,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:04,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:06,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:08,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:08,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:10,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:39:10,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:39:10,401 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:39:12,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:13,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:14,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:16,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:17,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:39:17,394 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:39:17,395 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:39:17,398 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:39:18,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:18,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:39:19,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:19,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:20,131 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:39:20,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:22,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:24,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:24,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:39:25,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:39:25,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:39:26,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:28,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:29,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:30,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:32,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:34,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:35,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:36,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:38,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:40,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:40,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:39:40,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:40,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:39:40,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:39:42,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:44,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:45,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:46,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:48,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:50,133 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:39:50,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:51,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:52,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:54,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:55,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:39:55,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:39:55,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:39:56,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:39:56,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:39:58,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:00,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:01,996 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:02,028 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:40:02,030 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:40:02,031 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:40:02,032 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:40:02,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:02,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:40:03,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:04,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:06,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:07,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:08,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:10,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:40:10,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:40:10,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:40:12,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:12,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:14,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:16,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:18,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:18,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:20,135 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:40:20,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:22,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:23,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:24,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:40:25,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:40:25,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:40:26,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:28,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:28,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:30,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:32,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:34,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:34,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:36,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:38,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:39,431 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:40,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:40,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:40:40,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:40:40,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:40:42,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:44,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:44,767 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:46,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:46,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:40:46,582 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:40:46,582 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:40:46,584 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:40:47,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:40:48,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:49,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:50,111 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:50,150 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:40:50,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:52,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:54,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:55,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:40:55,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:40:55,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:40:55,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:40:56,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:40:58,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:00,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:00,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:02,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:04,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:06,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:06,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:08,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:10,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:10,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:41:10,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:41:10,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:41:11,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:12,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:14,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:16,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:16,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:18,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:20,139 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:41:20,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:22,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:22,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:24,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:25,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:41:25,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:41:25,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:41:26,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:27,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:28,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:30,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:31,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:41:31,054 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:41:31,054 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:41:31,056 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:41:31,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:41:32,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:32,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:34,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:36,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:38,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:38,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:40,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:40,395 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:41:40,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:41:40,402 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:41:42,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:43,486 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:44,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:46,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:48,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:48,850 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:50,141 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:41:50,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:52,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:54,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:41:54,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:55,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:41:55,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:41:55,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:41:56,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:58,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:41:59,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:00,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:02,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:04,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:04,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:06,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:08,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:10,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:10,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:10,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:42:10,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:42:10,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:42:12,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:14,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:15,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:15,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:42:15,624 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:42:15,624 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:42:15,626 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:42:16,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:16,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:42:18,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:20,144 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:42:20,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:20,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:22,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:24,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:25,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:42:25,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:42:25,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:42:26,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:26,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:28,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:30,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:31,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:32,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:34,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:36,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:37,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:38,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:40,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:40,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:42:40,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:42:40,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:42:42,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:42,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:44,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:46,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:47,720 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:48,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:50,158 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:42:50,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:52,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:53,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:42:54,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:55,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:42:55,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:42:55,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:42:56,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:58,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:42:58,413 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:00,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:43:00,229 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:43:00,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:43:00,231 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:43:00,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:00,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:43:02,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:03,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:04,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:06,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:08,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:09,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:10,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:10,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:43:10,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:43:10,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:43:12,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:14,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:14,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:16,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:18,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:19,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:20,148 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:43:20,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:22,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:24,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:25,142 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:25,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:43:25,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:43:25,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:43:26,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:28,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:30,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:30,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:32,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:34,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:35,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:36,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:38,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:40,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:43:40,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:43:40,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:43:40,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:41,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:42,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:44,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:44,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:43:44,732 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:43:44,733 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:43:44,734 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:43:45,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:43:46,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:46,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:48,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:50,150 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:43:50,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:51,827 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:52,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:54,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:55,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:43:55,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:43:55,403 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:43:56,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:43:57,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:43:58,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:00,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:02,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:02,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:04,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:06,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:07,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:08,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:10,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:44:10,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:44:10,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:44:10,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:12,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:13,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:14,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:16,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:18,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:18,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:20,152 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:44:20,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:22,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:23,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:24,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:25,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:44:25,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:44:25,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:44:26,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:28,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:29,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:29,331 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:44:29,333 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:44:29,333 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:44:29,334 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:44:29,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:44:30,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:32,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:34,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:34,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:36,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:38,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:39,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:40,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:44:40,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:44:40,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:44:40,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:42,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:44,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:45,293 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:46,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:48,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:50,154 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:44:50,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:50,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:52,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:54,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:55,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:44:55,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:44:55,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:44:55,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:44:56,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:44:58,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:00,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:01,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:02,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:04,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:06,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:06,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:08,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:10,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:45:10,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:45:10,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:45:10,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:12,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:12,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:13,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:45:13,915 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:45:13,916 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:45:13,917 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:45:14,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:14,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:45:16,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:17,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:18,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:20,156 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:45:20,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:22,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:22,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:24,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:25,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:45:25,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:45:25,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:45:26,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:28,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:28,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:30,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:32,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:33,471 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:34,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:36,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:38,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:38,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:40,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:45:40,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:45:40,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:45:40,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:42,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:44,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:44,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:46,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:48,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:49,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:50,159 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:45:50,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:52,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:54,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:54,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:45:55,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:45:55,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:45:55,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:45:56,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:58,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:45:58,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:45:58,594 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:45:58,594 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:45:58,596 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:45:59,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:46:00,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:00,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:02,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:04,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:05,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:06,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:08,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:10,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:46:10,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:46:10,404 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:46:10,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:11,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:12,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:14,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:16,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:16,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:18,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:20,160 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:46:20,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:21,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:22,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:24,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:25,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:46:25,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:46:25,416 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:46:26,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:27,116 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:28,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:30,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:32,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:32,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:34,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:36,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:37,786 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:38,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:40,407 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:46:40,414 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:46:40,414 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:46:40,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:42,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:43,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:43,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:46:43,180 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:46:43,180 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:46:43,182 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:46:43,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:46:44,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:44,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:46:44,746 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:46:44,746 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:46:44,749 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:46:45,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:46:46,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:48,670 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:50,173 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:46:50,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:46:54,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:46:55,735 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:46:55,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:46:55,981 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:46:59,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:00,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:02,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:04,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:04,855 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:06,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:08,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:10,195 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:10,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:47:10,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:47:10,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:47:10,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:12,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:14,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:15,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:16,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:18,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:20,165 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:47:20,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:21,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:22,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:24,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:25,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:47:25,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:47:25,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:47:26,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:26,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:28,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:30,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:31,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:32,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:34,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:36,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:37,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:38,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:40,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:47:40,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:47:40,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:47:40,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:42,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:42,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:43,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 18:47:43,313 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 18:47:43,313 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 18:47:43,315 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 18:47:43,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 18:47:44,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:46,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:47,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:48,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:50,167 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 18:47:50,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:52,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:53,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:54,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:55,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:47:55,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:47:55,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:47:56,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:47:58,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:47:58,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:00,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:02,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:03,973 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:04,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:06,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:08,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 18:48:09,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 18:48:10,416 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 18:48:10,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 18:48:10,451 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 18:48:10,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log