diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,2028 +1,9520 @@ -2024-04-24 16:43:24,540 INFO StreamThr :1854781 [internal.py:wandb_internal():86] W&B internal server running at pid: 1854781, started at: 2024-04-24 16:43:24.539371 -2024-04-24 16:43:24,542 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status -2024-04-24 16:43:24,545 INFO WriterThread:1854781 [datastore.py:open_for_write():85] open: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/run-xfbnm7qo.wandb -2024-04-24 16:43:24,546 DEBUG SenderThread:1854781 [sender.py:send():382] send: header -2024-04-24 16:43:24,571 DEBUG SenderThread:1854781 [sender.py:send():382] send: run -2024-04-24 16:43:24,843 INFO SenderThread:1854781 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files -2024-04-24 16:43:24,843 INFO SenderThread:1854781 [sender.py:_start_run_threads():1136] run started: xfbnm7qo with start time 1713977004.542006 -2024-04-24 16:43:24,850 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: check_version -2024-04-24 16:43:24,850 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: check_version -2024-04-24 16:43:24,946 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: run_start -2024-04-24 16:43:25,004 DEBUG HandlerThread:1854781 [system_info.py:__init__():32] System info init -2024-04-24 16:43:25,004 DEBUG HandlerThread:1854781 [system_info.py:__init__():47] System info init done -2024-04-24 16:43:25,004 INFO HandlerThread:1854781 [system_monitor.py:start():194] Starting system monitor -2024-04-24 16:43:25,004 INFO SystemMonitor:1854781 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-24 16:43:25,004 INFO HandlerThread:1854781 [system_monitor.py:probe():214] Collecting system info -2024-04-24 16:43:25,005 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started cpu monitoring -2024-04-24 16:43:25,005 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started disk monitoring -2024-04-24 16:43:25,006 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started gpu monitoring -2024-04-24 16:43:25,006 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started memory monitoring -2024-04-24 16:43:25,007 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started network monitoring -2024-04-24 16:43:25,057 DEBUG HandlerThread:1854781 [system_info.py:probe():196] Probing system -2024-04-24 16:43:25,060 DEBUG HandlerThread:1854781 [system_info.py:_probe_git():181] Probing git -2024-04-24 16:43:25,080 DEBUG HandlerThread:1854781 [system_info.py:_probe_git():189] Probing git done -2024-04-24 16:43:25,080 DEBUG HandlerThread:1854781 [system_info.py:probe():244] Probing system done -2024-04-24 16:43:25,080 DEBUG HandlerThread:1854781 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.5', 'heartbeatAt': '2024-04-24T16:43:25.058035', 'startedAt': '2024-04-24T16:43:24.523748', 'docker': None, 'cuda': None, 'args': ('config_full.yaml',), 'state': 'running', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/run_sft.py', 'codePathLocal': 'run_sft.py', 'codePath': 'run_sft.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-zephyr-1.5b-ssft-ultrachat', 'commit': 'cbea69c6b95c970317a1e47c3f614b55b33f8ed9'}, 'email': None, 'root': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat', 'host': 'ip-26-0-162-233', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2729.8387291666663, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.161, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3584.12, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.175, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.329, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3596.81, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.102, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3596.611, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.198, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 59.25613021850586}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855270385742}} -2024-04-24 16:43:25,081 INFO HandlerThread:1854781 [system_monitor.py:probe():224] Finished collecting system info -2024-04-24 16:43:25,081 INFO HandlerThread:1854781 [system_monitor.py:probe():227] Publishing system info -2024-04-24 16:43:25,081 DEBUG HandlerThread:1854781 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment -2024-04-24 16:43:25,082 DEBUG HandlerThread:1854781 [system_info.py:_save_pip():68] Saving pip packages done -2024-04-24 16:43:25,083 DEBUG HandlerThread:1854781 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment -2024-04-24 16:43:25,845 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/conda-environment.yaml -2024-04-24 16:43:25,845 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/requirements.txt -2024-04-24 16:43:30,500 DEBUG HandlerThread:1854781 [system_info.py:_save_conda():87] Saving conda packages done -2024-04-24 16:43:30,502 INFO HandlerThread:1854781 [system_monitor.py:probe():229] Finished publishing system info -2024-04-24 16:43:30,525 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:43:30,525 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: keepalive -2024-04-24 16:43:30,525 DEBUG SenderThread:1854781 [sender.py:send():382] send: files -2024-04-24 16:43:30,526 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now -2024-04-24 16:43:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:43:30,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:43:30,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:43:30,621 DEBUG SenderThread:1854781 [sender.py:send():382] send: telemetry -2024-04-24 16:43:30,622 DEBUG SenderThread:1854781 [sender.py:send():382] send: config -2024-04-24 16:43:30,622 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric -2024-04-24 16:43:30,623 DEBUG SenderThread:1854781 [sender.py:send():382] send: telemetry -2024-04-24 16:43:30,623 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric -2024-04-24 16:43:30,623 WARNING SenderThread:1854781 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) -2024-04-24 16:43:30,623 DEBUG SenderThread:1854781 [sender.py:send():382] send: telemetry -2024-04-24 16:43:30,795 INFO wandb-upload_0:1854781 [upload_job.py:push():131] Uploaded file /tmp/tmp2ljkzqevwandb/om53ihff-wandb-metadata.json -2024-04-24 16:43:30,849 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/conda-environment.yaml -2024-04-24 16:43:30,849 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-metadata.json -2024-04-24 16:43:30,849 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:32,851 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:33,571 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:43:33,573 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric -2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric -2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric -2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric -2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:43:33,576 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:43:33,852 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:43:34,854 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:35,578 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:43:35,855 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:36,855 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:38,858 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:40,807 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:43:40,860 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:42,861 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:44,863 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:43:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:43:45,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:43:46,660 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:43:46,865 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:48,867 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:50,869 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:52,488 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:43:52,871 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:54,873 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:56,875 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:43:58,187 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:43:58,877 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/config.yaml -2024-04-24 16:43:58,878 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:00,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:44:00,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:44:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:44:00,879 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:02,881 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:03,895 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:04,883 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:06,885 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:08,887 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:09,580 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:10,889 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:12,891 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:14,893 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:15,263 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:15,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:44:15,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:44:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:44:16,895 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:18,897 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:19,965 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:44:19,967 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:44:19,967 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:44:19,969 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:44:20,899 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:44:20,899 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:20,970 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:21,900 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:22,901 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:24,903 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:25,007 DEBUG SystemMonitor:1854781 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-24 16:44:25,011 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:44:26,624 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:26,905 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:28,907 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:44:30,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:44:30,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:44:30,908 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:32,318 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:32,910 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:34,912 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:36,914 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:38,005 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:38,916 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:40,918 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:42,920 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:43,687 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:44,922 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:44:45,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:44:45,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:44:46,924 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:48,925 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:49,361 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:50,927 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:52,929 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:54,931 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:55,012 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:44:55,013 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:44:56,933 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:44:58,935 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:00,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:45:00,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:45:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:45:00,698 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:00,937 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:02,939 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:04,941 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:06,452 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:06,942 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:07,375 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:45:07,377 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:45:07,377 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:45:07,379 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:45:07,944 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:45:08,945 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:09,946 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:10,947 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:12,142 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:12,949 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:14,951 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:15,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:45:15,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:45:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:45:16,953 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:17,819 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:18,955 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:20,957 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:22,958 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:23,488 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:24,960 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:25,023 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:45:26,962 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:28,964 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:29,122 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:45:30,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:45:30,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:45:30,966 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:32,968 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:34,739 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:34,970 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:36,972 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:38,973 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:40,343 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:40,975 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:42,977 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:44,979 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:45:45,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:45:45,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:45:45,667 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:46,981 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:48,983 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:50,985 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:51,602 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:52,987 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:54,402 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:45:54,404 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:45:54,404 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:45:54,405 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:45:54,989 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:45:54,990 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:55,016 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:45:55,990 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:56,991 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:45:57,250 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:45:58,993 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:00,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:46:00,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:46:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:46:00,995 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:02,892 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:02,997 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:04,999 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:07,000 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:08,529 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:09,002 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:11,004 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:13,006 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:14,157 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:15,008 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:15,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:46:15,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:46:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:46:17,010 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:19,012 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:19,805 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:21,014 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:23,016 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:25,018 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:46:25,019 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:25,019 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:27,021 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:29,023 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:30,090 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:30,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:46:30,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:46:30,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:46:31,025 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:33,027 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:35,029 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:35,728 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:37,031 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:39,033 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:41,035 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:41,336 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:41,362 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:46:41,364 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:46:41,364 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:46:41,366 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:46:42,036 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:46:43,037 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:44,038 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:45,039 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:45,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:46:45,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:46:45,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:46:46,613 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:47,041 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:49,043 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:51,045 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:51,657 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:53,047 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:55,020 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:46:55,049 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:57,051 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:46:57,256 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:46:59,052 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:00,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:47:00,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:47:00,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:47:01,054 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:02,864 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:03,056 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:05,058 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:07,060 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:08,484 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:09,062 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:11,064 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:13,066 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:14,088 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:15,068 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:15,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:47:15,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:47:15,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:47:17,070 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:19,072 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:19,691 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:21,073 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:23,075 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:25,022 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:47:25,023 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:25,077 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:27,079 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:28,045 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:47:28,047 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:47:28,047 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:47:28,049 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:47:28,081 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:47:29,082 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:30,083 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:30,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:47:30,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:47:30,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:47:30,699 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:31,084 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:33,085 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:35,087 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:36,463 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:37,089 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:39,091 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:41,093 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:42,044 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:43,095 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:45,097 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:45,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:47:45,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:47:45,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:47:47,099 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:47,612 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:49,101 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:51,102 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:53,104 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:53,199 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:55,024 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:47:55,106 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:57,108 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:47:58,737 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:47:59,110 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:00,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:48:00,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:48:00,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:48:01,112 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:03,114 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:04,295 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:05,115 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:07,118 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:09,120 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:09,834 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:11,121 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:13,123 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:14,418 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:48:14,419 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:48:14,420 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:48:14,422 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:48:15,126 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:48:15,126 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:15,422 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:15,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:48:15,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:48:15,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:48:16,127 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:17,128 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:19,130 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:20,943 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:21,131 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:23,133 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:25,026 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:48:25,135 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:26,510 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:27,137 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:29,139 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:30,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:48:30,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:48:30,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:48:31,141 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:32,087 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:33,143 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:35,145 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:37,147 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:37,670 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:39,149 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:41,151 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:43,153 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:43,245 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:45,154 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:45,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:48:45,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:48:45,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:48:47,156 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:48,827 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:49,158 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:51,161 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:53,162 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:54,384 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:48:55,040 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:48:55,164 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:57,166 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:59,168 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:48:59,926 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:49:00,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:49:00,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:49:00,822 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:49:00,823 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:49:00,823 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:49:00,825 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:49:01,170 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:49:01,171 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:02,171 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:03,172 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:05,174 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:05,484 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:07,176 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:09,178 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:11,052 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:11,180 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:13,182 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:15,184 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:49:15,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:49:15,536 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:49:16,624 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:17,186 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:19,188 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:21,190 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:22,191 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:23,192 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:25,030 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:49:25,194 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:27,196 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:27,746 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:29,198 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:30,570 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:49:30,571 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:49:30,571 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:49:31,200 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:33,202 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:33,300 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:35,203 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:37,205 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:38,828 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:39,207 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:41,209 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:43,211 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:44,336 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:45,213 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:45,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:49:45,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:49:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:49:47,058 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:49:47,059 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:49:47,060 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:49:47,061 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:49:47,215 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:49:47,215 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:48,216 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:49,217 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:49,858 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:51,219 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:53,221 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:55,032 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:49:55,033 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:49:55,223 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:57,225 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:49:59,227 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:00,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:50:00,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:50:00,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:50:00,701 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:01,229 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:03,231 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:05,233 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:06,467 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:07,235 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:09,236 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:11,238 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:12,009 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:13,240 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:15,242 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:15,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:50:15,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:50:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:50:17,244 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:17,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:19,246 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:21,248 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:23,102 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:23,250 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:25,034 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:50:25,252 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:27,254 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:28,648 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:29,256 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:30,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:50:30,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:50:30,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:50:31,258 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:33,218 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:50:33,219 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:50:33,220 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:50:33,221 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:50:33,260 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:50:33,261 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:34,222 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:35,262 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:36,263 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:37,264 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:39,266 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:39,743 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:41,268 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:43,270 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:45,270 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:45,272 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:45,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:50:45,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:50:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:50:47,274 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:49,276 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:50,784 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:51,278 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:53,280 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:55,072 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:50:55,281 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:56,317 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:50:57,283 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:50:59,285 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:00,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:51:00,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:51:00,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:51:01,287 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:01,840 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:03,289 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:05,291 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:07,293 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:07,360 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:09,295 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:11,296 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:12,884 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:13,299 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:15,300 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:15,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:51:15,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:51:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:51:17,302 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:18,392 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:19,268 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:51:19,269 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:51:19,270 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:51:19,271 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:51:19,305 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:51:19,305 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:21,307 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:22,308 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:23,309 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:23,942 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:25,038 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:51:25,311 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:27,312 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:29,314 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:29,487 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:30,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:51:30,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:51:30,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:51:31,317 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:33,319 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:34,977 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:35,320 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:37,322 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:39,324 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:40,471 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:41,326 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:43,328 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:45,330 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:45,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:51:45,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:51:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:51:45,662 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:47,331 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:49,333 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:51,335 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:51,427 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:53,337 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:55,040 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:51:55,339 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:56,920 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:51:57,341 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:51:59,343 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:00,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:52:00,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:52:00,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:52:01,346 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:02,415 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:03,348 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:05,117 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:52:05,118 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:52:05,118 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:52:05,120 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:52:05,351 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:52:05,351 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:06,352 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:07,354 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:07,925 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:09,358 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:11,364 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:13,367 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:13,429 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:15,371 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:15,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:52:15,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:52:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:52:17,375 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:18,951 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:19,380 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:21,385 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:23,389 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:24,470 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:25,042 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:52:25,393 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:27,398 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:29,409 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:29,954 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:30,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:52:30,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:52:30,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:52:31,413 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:33,418 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:35,424 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:35,435 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:37,426 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:39,428 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:40,923 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:41,431 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:43,433 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:45,435 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:45,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:52:45,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:52:45,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:52:46,692 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:47,437 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:49,439 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:50,949 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:52:50,951 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:52:50,951 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:52:50,953 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:52:51,442 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:52:51,442 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:51,953 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:52,443 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:53,446 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:55,044 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:52:55,448 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:57,431 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:52:57,450 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:52:59,452 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:00,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:53:00,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:53:00,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:53:01,453 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:02,948 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:03,455 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:05,457 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:07,459 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:08,459 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:09,461 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:11,463 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:13,465 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:13,977 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:15,467 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:15,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:53:15,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:53:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:53:17,469 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:19,472 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:19,488 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:21,685 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:23,688 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:25,005 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:25,046 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:53:25,691 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:27,698 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:29,702 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:30,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:53:30,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:53:30,568 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:53:31,704 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:33,706 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:35,708 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:36,033 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:36,891 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:53:36,894 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:53:36,894 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:53:36,905 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:53:37,710 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:53:37,711 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:39,712 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:41,547 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:41,714 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:43,716 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:45,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:53:45,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:53:45,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:53:45,718 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:47,029 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:47,720 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:49,722 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:51,724 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:52,531 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:53,726 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:55,048 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:53:55,728 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:57,730 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:53:58,028 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:53:59,732 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:00,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:54:00,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:54:00,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:54:01,734 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:03,525 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:03,736 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:05,737 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:07,739 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:09,010 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:09,741 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:11,743 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:13,745 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:14,495 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:15,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:54:15,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:54:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:54:15,747 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:17,749 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:19,751 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:19,951 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:21,753 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:22,601 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:54:22,603 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:54:22,603 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:54:22,605 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:54:22,755 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:54:23,755 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:25,060 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:54:25,060 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:25,757 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:27,759 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:29,761 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:30,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:54:30,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:54:30,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:54:30,625 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:31,763 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:33,765 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:35,767 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:36,349 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:37,769 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:39,770 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:41,772 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:41,837 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:43,774 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:45,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:54:45,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:54:45,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:54:45,776 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:47,336 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:47,778 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:49,780 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:51,781 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:52,840 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:53,783 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:55,053 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:54:55,785 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:57,787 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:54:58,343 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:54:59,789 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:00,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:55:00,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:55:00,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:55:01,791 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:03,792 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:03,853 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:05,794 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:07,796 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:08,387 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:55:08,388 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:55:08,389 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:55:08,390 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:55:08,798 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:55:09,391 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:09,799 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:11,801 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:13,802 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:14,878 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:15,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:55:15,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:55:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:55:15,804 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:17,806 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:19,808 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:20,367 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:21,810 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:23,812 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:25,055 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:55:25,813 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:26,055 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:27,815 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:29,817 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:30,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:55:30,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:55:30,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:55:31,656 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:31,819 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:33,821 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:35,823 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:36,734 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:37,824 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:39,826 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:41,828 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:42,203 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:43,830 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:45,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:55:45,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:55:45,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:55:45,832 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:47,668 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:47,834 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:49,836 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:51,838 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:53,139 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:53,839 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:54,003 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:55:54,005 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:55:54,005 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:55:54,007 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:55:54,841 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:55:55,056 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:55:55,842 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:57,844 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:55:58,637 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:55:59,846 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:00,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:56:00,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:56:00,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:56:01,848 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:03,850 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:04,136 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:05,852 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:07,854 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:09,643 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:09,855 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:11,857 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:13,859 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:15,120 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:15,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:56:15,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:56:15,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:56:15,861 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:17,863 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:19,865 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:20,612 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:21,867 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:23,869 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:25,068 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:56:25,871 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:26,117 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:27,873 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:29,875 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:30,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:56:30,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:56:30,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:56:31,717 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:31,877 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:33,879 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:35,881 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:37,074 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:37,883 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:39,746 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:56:39,748 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:56:39,748 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:56:39,750 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:56:39,885 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:56:39,886 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:41,887 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:42,546 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:43,889 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:45,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:56:45,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:56:45,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:56:45,891 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:47,893 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:48,021 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:49,895 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:51,897 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:53,490 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:53,898 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:55,060 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:56:55,900 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:57,902 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:56:58,986 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:56:59,904 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:00,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:57:00,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:57:00,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:57:01,906 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:03,908 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:04,466 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:05,910 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:07,912 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:09,914 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:09,952 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:11,916 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:13,917 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:15,446 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:15,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:57:15,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:57:15,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:57:15,919 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:17,921 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:19,923 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:20,944 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:21,925 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:23,927 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:25,062 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:57:25,467 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:57:25,469 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:57:25,469 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:57:25,470 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:57:25,929 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:57:25,930 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:26,471 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:27,931 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:29,933 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:30,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:57:30,556 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:57:30,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:57:30,934 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:31,912 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:32,936 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:34,938 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:36,940 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:37,373 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:38,942 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:40,944 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:42,825 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:42,945 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:44,947 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:45,576 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:57:45,576 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:57:45,576 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:57:46,949 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:48,286 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:48,951 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:50,953 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:52,955 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:53,746 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:57:54,956 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:55,064 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:57:56,958 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:58,960 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:57:59,216 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:00,563 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:58:00,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:58:00,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:58:00,962 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:02,964 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:04,703 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:04,966 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:06,968 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:08,970 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:10,203 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:10,972 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:11,053 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:58:11,055 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:58:11,055 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:58:11,057 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:58:11,974 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:58:11,974 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:12,975 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:14,977 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:58:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:15,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:58:15,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:58:16,978 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:18,980 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:20,982 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:21,213 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:22,984 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:24,986 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:25,066 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:58:26,714 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:26,988 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:28,990 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:30,563 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:58:30,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:58:30,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:58:30,992 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:32,197 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:32,994 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:34,996 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:36,999 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:37,677 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:39,001 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:41,002 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:43,004 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:43,163 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:45,006 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:58:45,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:58:45,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:58:47,009 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:48,630 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:49,011 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:51,013 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:53,015 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:54,110 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:58:55,016 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:55,070 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:58:56,778 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:58:56,779 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:58:56,779 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:58:56,782 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:58:57,019 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:58:57,019 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:58,020 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:58:59,789 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:01,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:59:01,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:59:01,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:59:03,025 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:05,171 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:10,171 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:13,035 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:15,037 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:16,015 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:16,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:59:16,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:59:16,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:59:17,038 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:19,040 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:21,042 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:21,480 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:23,044 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:25,046 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:25,072 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:59:26,944 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:27,048 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:29,049 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:31,051 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:31,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:59:31,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:59:31,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:59:32,434 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:33,053 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:35,055 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:37,057 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:37,910 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:39,059 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:41,060 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:43,062 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:43,396 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:45,064 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:46,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 16:59:46,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 16:59:46,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 16:59:47,066 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:48,885 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:49,068 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:51,070 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:53,071 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:54,358 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 16:59:55,074 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:55,083 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 16:59:57,039 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 16:59:57,040 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 16:59:57,040 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 16:59:57,042 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 16:59:57,077 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 16:59:57,077 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:58,078 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:59,079 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 16:59:59,841 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:01,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:00:01,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:00:01,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:00:01,081 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:03,082 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:05,084 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:05,331 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:07,086 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:09,088 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:10,811 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:11,090 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:13,092 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:15,094 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:16,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:00:16,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:16,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:00:16,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:00:17,095 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:19,097 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:21,099 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:21,765 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:23,101 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:25,075 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:00:25,103 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:27,104 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:27,223 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:29,106 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:31,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:00:31,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:00:31,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:00:31,108 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:32,691 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:33,110 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:35,112 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:37,114 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:38,137 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:39,115 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:41,117 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:42,630 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:00:42,632 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:00:42,632 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:00:42,633 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:00:43,120 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:00:43,120 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:43,634 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:44,121 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:45,122 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:46,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:00:46,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:00:46,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:00:47,124 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:49,087 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:49,125 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:51,127 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:53,129 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:54,577 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:00:55,078 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:00:55,131 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:57,132 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:00:59,134 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:00,073 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:01,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:01:01,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:01:01,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:01:01,136 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:03,138 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:05,140 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:05,551 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:07,141 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:09,143 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:11,043 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:11,145 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:13,147 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:15,149 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:16,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:01:16,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:16,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:01:16,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:01:17,150 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:19,152 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:21,154 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:21,997 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:23,156 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:25,080 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:01:25,158 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:27,159 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:27,475 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:28,324 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:01:28,326 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:01:28,326 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:01:28,328 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:01:29,162 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:01:29,162 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:31,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:01:31,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:01:31,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:01:31,164 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:32,941 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:33,166 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:35,167 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:37,169 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:38,410 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:39,171 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:41,173 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:43,175 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:43,860 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:45,177 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:46,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:01:46,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:01:46,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:01:47,179 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:49,181 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:49,318 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:51,182 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:53,184 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:54,767 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:01:55,082 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:01:55,197 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:57,198 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:01:59,201 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:00,230 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:01,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:02:01,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:02:01,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:02:01,204 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:03,206 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:05,209 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:05,698 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:07,211 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:09,213 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:11,171 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:11,215 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:13,220 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:13,846 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:02:13,847 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:02:13,848 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:02:13,849 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:02:14,221 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:02:15,223 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:16,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:02:16,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:02:16,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:02:17,163 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:17,226 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:19,228 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:21,229 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:22,973 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:23,231 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:25,084 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:02:25,233 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:27,235 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:28,434 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:29,236 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:31,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:02:31,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:02:31,069 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:02:31,238 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:33,240 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:33,880 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:35,242 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:37,244 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:39,246 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:39,315 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:41,248 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:43,250 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:44,749 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:45,252 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:46,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:02:46,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:02:46,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:02:47,253 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:49,255 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:50,187 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:51,261 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:53,266 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:55,085 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:02:55,268 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:55,635 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:02:57,270 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:59,271 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:02:59,303 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:02:59,304 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:02:59,305 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:02:59,306 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:03:00,273 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:03:01,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:03:01,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:01,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:03:01,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:03:01,274 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:03,276 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:05,278 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:06,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:07,280 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:09,282 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:11,283 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:12,022 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:13,285 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:15,287 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:16,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:03:16,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:03:16,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:03:17,187 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:17,289 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:19,291 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:21,293 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:22,960 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:23,458 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:25,098 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:03:25,461 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:27,463 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:28,411 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:29,465 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:31,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:03:31,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:03:31,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:03:31,467 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:33,469 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:33,847 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:35,471 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:37,472 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:39,294 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:39,475 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:41,476 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:43,478 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:44,731 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:44,758 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:03:44,759 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:03:44,760 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:03:44,762 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:03:45,481 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:03:45,481 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:46,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:03:46,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:03:46,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:03:47,483 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:49,485 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:50,180 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:51,487 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:53,489 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:55,089 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:03:55,490 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:55,642 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:03:57,492 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:03:59,494 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:01,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:04:01,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:01,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:04:01,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:04:01,496 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:03,498 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:05,500 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:06,581 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:07,502 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:09,504 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:11,506 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:12,057 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:13,508 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:15,510 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:16,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:04:16,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:04:16,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:04:17,185 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:17,512 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:19,514 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:21,516 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:22,186 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:23,518 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:25,137 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:04:25,520 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:27,521 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:27,626 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:29,523 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:30,285 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:04:30,287 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:04:30,287 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:04:30,289 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:04:30,525 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:04:31,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:04:31,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:04:31,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:04:31,526 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:33,081 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:33,528 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:35,530 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:37,532 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:38,516 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:39,534 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:41,536 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:43,537 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:43,949 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:45,539 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:46,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:04:46,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:04:46,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:04:49,403 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:51,545 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:53,547 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:54,849 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:04:55,139 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:04:55,549 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:57,551 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:04:59,553 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:00,315 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:01,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:05:01,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:05:01,070 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:05:01,554 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:03,556 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:05,558 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:05,789 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:07,560 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:09,562 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:11,263 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:11,564 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:13,565 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:15,567 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:15,764 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:05:15,766 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:05:15,766 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:05:15,768 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:05:16,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:05:16,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:05:16,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:05:16,569 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:05:17,237 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:17,570 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:19,572 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:21,574 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:23,022 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:23,576 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:25,151 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:05:25,577 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:27,579 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:28,456 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:29,581 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:31,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:05:31,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:05:31,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:05:31,583 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:33,585 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:33,904 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:35,586 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:37,588 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:39,346 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:39,590 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:41,592 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:43,594 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:44,793 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:45,595 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:46,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:05:46,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:05:46,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:05:47,597 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:49,599 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:50,247 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:51,601 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:53,603 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:55,143 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:05:55,605 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:55,711 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:05:57,607 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:05:59,609 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:01,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:06:01,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:01,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:06:01,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:06:01,213 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:06:01,214 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:06:01,214 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:06:01,215 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:06:01,611 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:06:01,612 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:03,613 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:05,615 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:06,658 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:07,617 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:09,619 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:11,621 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:12,136 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:13,623 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:15,625 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:16,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:06:16,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:06:16,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:06:17,183 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:17,627 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:19,629 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:21,631 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:22,216 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:23,633 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:25,145 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:06:25,635 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:27,637 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:27,653 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:29,639 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:31,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:06:31,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:06:31,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:06:31,641 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:33,093 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:33,643 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:35,644 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:37,646 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:38,540 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:39,648 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:41,650 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:43,652 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:43,995 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:45,654 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:46,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:06:46,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:06:46,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:06:46,682 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:06:46,684 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:06:46,684 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:06:46,685 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:06:47,657 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:06:47,657 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:49,473 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:49,658 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:51,660 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:53,662 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:54,942 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:06:55,147 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:06:55,664 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:57,666 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:06:59,668 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:00,416 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:01,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:07:01,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:07:01,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:07:01,670 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:03,671 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:05,673 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:05,879 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:07,675 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:09,677 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:11,333 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:11,679 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:13,681 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:15,683 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:16,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:07:16,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:07:16,072 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:07:17,147 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:17,684 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:19,686 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:21,689 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:22,240 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:23,691 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:25,149 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:07:25,693 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:27,695 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:27,696 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:29,697 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:31,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:07:31,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:07:31,072 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:07:31,699 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:32,194 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:07:32,195 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:07:32,195 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:07:32,197 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:07:32,701 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:07:33,198 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:33,702 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:35,704 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:37,706 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:38,625 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:39,708 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:41,710 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:43,711 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:44,099 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:45,713 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:46,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:07:46,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:07:46,071 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:07:47,715 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:49,550 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:49,717 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:51,719 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:53,721 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:54,997 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:07:55,151 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:07:55,723 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:57,725 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:07:59,727 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:00,425 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:01,070 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:08:01,071 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:08:01,072 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:08:01,729 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:03,730 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:05,732 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:05,845 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:07,734 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:09,736 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:11,267 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:11,737 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:13,739 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:15,741 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:08:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:08:16,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:08:16,325 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:17,743 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:17,774 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:08:17,776 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:08:17,776 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:08:17,778 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:08:18,745 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:08:19,746 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:21,375 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:21,748 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:23,749 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:25,152 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:08:25,751 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:26,832 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:27,753 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:29,755 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:31,208 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:08:31,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:08:31,209 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:08:31,757 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:32,288 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:33,759 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:35,761 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:37,718 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:37,763 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:39,765 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:41,766 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:43,159 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:43,768 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:45,770 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:46,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:08:46,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:08:46,209 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:08:47,772 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:48,604 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:49,774 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:51,776 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:53,778 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:54,078 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:55,164 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:08:55,779 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:57,781 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:08:59,549 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:08:59,783 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:01,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:09:01,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:09:01,209 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:09:01,785 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:03,222 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:09:03,223 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:09:03,224 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:09:03,225 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:09:03,788 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:09:03,788 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:05,021 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:05,790 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:07,791 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:09,793 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:10,484 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:11,797 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:13,799 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:15,801 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:15,942 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:16,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:09:16,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:09:16,209 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:09:17,803 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:19,805 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:21,402 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:21,806 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:23,808 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:25,157 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:09:25,810 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:26,857 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:27,812 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:29,814 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:31,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:09:31,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:09:31,209 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:09:31,816 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:32,301 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:33,818 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:35,820 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:37,770 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:37,822 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:39,824 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:41,826 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:43,248 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:43,828 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:45,830 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:46,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:09:46,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:09:46,209 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:09:47,832 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:48,712 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:48,738 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:09:48,740 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:09:48,740 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:09:48,742 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:09:48,834 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:09:49,835 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:51,836 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:53,838 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:54,161 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:55,159 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:09:55,840 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:57,842 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:09:59,580 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:09:59,844 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:01,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:10:01,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:10:01,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:10:01,846 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:03,848 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:05,004 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:05,850 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:07,852 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:09,854 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:10,425 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:11,856 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:13,858 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:15,860 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:15,867 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:16,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:10:16,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:10:16,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:10:17,863 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:19,865 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:21,321 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:21,867 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:23,869 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:25,161 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:10:25,871 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:26,781 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:27,873 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:29,875 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:31,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:10:31,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:10:31,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:10:31,877 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:32,243 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:33,879 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:34,093 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:10:34,095 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:10:34,095 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:10:34,097 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:10:34,880 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:10:35,882 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:37,715 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:37,884 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:39,886 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:41,888 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:43,344 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:43,890 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:45,892 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:46,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:10:46,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:10:46,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:10:47,894 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:48,808 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:49,896 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:51,898 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:53,901 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:54,269 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:55,162 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:10:55,903 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:57,906 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:10:59,717 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:10:59,908 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:01,209 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:11:01,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:11:01,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:11:01,911 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:03,913 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:05,173 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:05,916 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:07,918 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:09,920 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:10,630 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:11,922 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:13,924 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:15,927 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:16,108 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:11:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:11:16,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:11:17,930 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:19,775 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:11:19,776 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:11:19,777 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:11:19,779 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:11:19,933 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:11:19,933 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:21,568 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:21,936 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:23,938 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:25,164 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:11:25,941 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:27,027 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:27,943 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:29,946 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:31,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:11:31,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:11:31,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:11:31,948 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:32,359 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:33,951 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:35,953 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:37,933 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:37,955 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:39,957 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:41,960 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:43,368 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:43,961 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:45,964 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:46,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:11:46,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:11:46,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:11:47,966 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:48,824 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:49,968 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:51,970 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:53,972 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:54,291 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:55,166 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:11:55,978 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:57,980 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:11:59,764 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:11:59,982 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:01,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:12:01,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:12:01,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:12:01,984 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:03,986 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:05,273 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:05,302 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:12:05,303 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:12:05,303 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:12:05,305 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:12:05,989 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:12:05,989 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:07,991 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:09,993 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:10,753 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:11,995 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:13,998 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:16,000 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:12:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:12:16,210 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:12:18,002 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:20,004 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:21,673 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:22,006 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:24,008 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:25,168 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:12:26,011 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:27,135 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:28,012 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:30,014 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:31,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:12:31,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:12:31,211 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:12:32,016 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:32,291 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:34,018 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:36,020 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:38,007 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:38,022 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:40,024 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:42,026 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:43,450 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:44,028 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:46,030 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:46,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:12:46,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:12:46,211 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:12:48,032 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:48,909 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:50,034 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:50,756 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:12:50,757 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:12:50,758 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:12:50,759 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:12:51,036 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:12:52,037 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:54,039 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:54,365 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:12:55,170 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:12:56,040 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:58,042 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:12:59,846 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:00,044 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:01,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:13:01,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:13:01,211 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:13:02,046 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:04,048 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:05,315 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:06,050 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:08,052 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:10,054 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:10,776 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:12,056 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:14,058 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:16,060 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:16,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:16,211 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:13:16,211 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:13:16,211 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:13:18,062 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:20,064 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:21,639 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:22,065 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:24,234 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:25,172 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:13:26,240 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:27,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:28,242 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:30,246 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:31,210 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:13:31,211 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:13:31,211 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:13:32,248 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:32,312 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:34,250 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:35,252 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:36,154 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:13:36,155 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:13:36,155 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:13:36,157 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:13:36,253 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:13:37,254 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:37,946 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:39,256 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:41,258 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:43,261 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:43,409 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:45,263 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:46,212 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:13:46,212 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:13:46,213 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:13:47,265 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:48,872 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:49,267 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:51,269 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:53,271 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:54,331 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:13:55,174 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:13:55,272 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:57,274 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:59,276 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:13:59,798 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:01,212 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:14:01,212 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:14:01,213 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:14:01,278 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:03,280 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:05,253 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:05,282 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:07,284 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:09,285 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:10,710 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:11,287 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:13,289 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:15,291 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:16,156 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:16,249 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:14:16,249 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:14:16,249 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:14:17,293 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:19,295 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:21,297 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:21,604 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:21,630 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:14:21,632 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:14:21,632 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:14:21,635 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:14:22,299 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:14:23,300 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:25,186 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:14:26,978 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:27,304 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:31,236 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:14:31,237 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:14:31,237 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:14:32,414 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:37,314 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:37,804 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:39,316 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:41,317 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:43,219 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:43,319 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:45,321 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:46,236 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:14:46,237 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:14:46,237 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:14:47,323 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:48,645 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:49,324 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:51,326 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:53,328 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:54,093 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:14:55,178 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:14:55,330 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:57,331 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:59,333 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:14:59,539 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:15:01,236 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:15:01,237 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:15:01,237 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:15:01,335 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:03,337 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:05,007 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:15:05,339 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:07,340 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:09,342 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:10,475 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:15:11,344 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:13,346 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:15,348 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:15,936 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:15:16,236 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:15:16,237 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:15:16,237 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:15:17,349 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:19,351 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:21,353 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:21,381 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:15:21,408 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history -2024-04-24 17:15:21,410 DEBUG SenderThread:1854781 [sender.py:send():382] send: history -2024-04-24 17:15:21,410 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record -2024-04-24 17:15:21,412 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2024-04-24 17:15:22,355 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json -2024-04-24 17:15:23,356 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:25,180 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats -2024-04-24 17:15:25,358 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:26,835 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report -2024-04-24 17:15:27,359 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:29,361 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log -2024-04-24 17:15:31,236 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-24 17:15:31,237 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status -2024-04-24 17:15:31,237 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status -2024-04-24 17:15:31,363 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-25 13:45:18,234 INFO StreamThr :156911 [internal.py:wandb_internal():86] W&B internal server running at pid: 156911, started at: 2024-04-25 13:45:18.233208 +2024-04-25 13:45:18,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status +2024-04-25 13:45:18,241 INFO WriterThread:156911 [datastore.py:open_for_write():87] open: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/run-etajcxpg.wandb +2024-04-25 13:45:18,243 DEBUG SenderThread:156911 [sender.py:send():379] send: header +2024-04-25 13:45:18,269 DEBUG SenderThread:156911 [sender.py:send():379] send: run +2024-04-25 13:45:18,501 INFO SenderThread:156911 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files +2024-04-25 13:45:18,501 INFO SenderThread:156911 [sender.py:_start_run_threads():1124] run started: etajcxpg with start time 1714052718.239809 +2024-04-25 13:45:18,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: check_version +2024-04-25 13:45:18,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: check_version +2024-04-25 13:45:18,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: run_start +2024-04-25 13:45:18,625 DEBUG HandlerThread:156911 [system_info.py:__init__():26] System info init +2024-04-25 13:45:18,625 DEBUG HandlerThread:156911 [system_info.py:__init__():41] System info init done +2024-04-25 13:45:18,625 INFO HandlerThread:156911 [system_monitor.py:start():194] Starting system monitor +2024-04-25 13:45:18,625 INFO SystemMonitor:156911 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-25 13:45:18,625 INFO HandlerThread:156911 [system_monitor.py:probe():214] Collecting system info +2024-04-25 13:45:18,625 INFO SystemMonitor:156911 [interfaces.py:start():190] Started cpu monitoring +2024-04-25 13:45:18,626 INFO SystemMonitor:156911 [interfaces.py:start():190] Started disk monitoring +2024-04-25 13:45:18,627 INFO SystemMonitor:156911 [interfaces.py:start():190] Started gpu monitoring +2024-04-25 13:45:18,627 INFO SystemMonitor:156911 [interfaces.py:start():190] Started memory monitoring +2024-04-25 13:45:18,627 INFO SystemMonitor:156911 [interfaces.py:start():190] Started network monitoring +2024-04-25 13:45:18,674 DEBUG HandlerThread:156911 [system_info.py:probe():150] Probing system +2024-04-25 13:45:18,677 DEBUG HandlerThread:156911 [system_info.py:_probe_git():135] Probing git +2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_info.py:_probe_git():143] Probing git done +2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_info.py:probe():198] Probing system done +2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.9', 'heartbeatAt': '2024-04-25T13:45:18.674598', 'startedAt': '2024-04-25T13:45:18.212334', 'docker': None, 'cuda': None, 'args': ('./config_full.yaml',), 'state': 'running', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/./run_sft.py', 'codePathLocal': 'run_sft.py', 'codePath': 'run_sft.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-zephyr-1.5b-ssft-ultrachat', 'commit': 'cbea69c6b95c970317a1e47c3f614b55b33f8ed9'}, 'email': None, 'root': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat', 'host': 'ip-26-0-167-177', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/alignment/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2725.4579687499986, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.164, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.955, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3596.13, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.91, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3592.057, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3267.919, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.091, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 3597.915, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}, {'current': 2649.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 58.59657669067383}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855155944824}} +2024-04-25 13:45:18,698 INFO HandlerThread:156911 [system_monitor.py:probe():224] Finished collecting system info +2024-04-25 13:45:18,698 INFO HandlerThread:156911 [system_monitor.py:probe():227] Publishing system info +2024-04-25 13:45:18,698 DEBUG HandlerThread:156911 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-25 13:45:19,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/conda-environment.yaml +2024-04-25 13:45:22,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/conda-environment.yaml +2024-04-25 13:45:22,523 DEBUG HandlerThread:156911 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-25 13:45:22,526 INFO HandlerThread:156911 [system_monitor.py:probe():229] Finished publishing system info +2024-04-25 13:45:22,561 DEBUG SenderThread:156911 [sender.py:send():379] send: files +2024-04-25 13:45:22,561 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-25 13:45:22,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: python_packages +2024-04-25 13:45:22,707 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: python_packages +2024-04-25 13:45:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:45:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:45:22,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:45:22,829 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry +2024-04-25 13:45:22,830 DEBUG SenderThread:156911 [sender.py:send():379] send: config +2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry +2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 13:45:22,831 WARNING SenderThread:156911 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-25 13:45:22,831 DEBUG SenderThread:156911 [sender.py:send():379] send: telemetry +2024-04-25 13:45:22,840 INFO wandb-upload_0:156911 [upload_job.py:push():131] Uploaded file /tmp/tmpcjtpmhukwandb/bsh2e287-wandb-metadata.json +2024-04-25 13:45:23,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:23,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/requirements.txt +2024-04-25 13:45:23,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-metadata.json +2024-04-25 13:45:23,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:45:25,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:26,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:45:26,068 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 13:45:26,069 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:45:26,070 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:45:26,071 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:45:26,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:45:27,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:28,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:29,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:45:31,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:33,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:34,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:45:35,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:37,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:45:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:45:37,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:45:39,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:45:39,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:41,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:43,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:45,235 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:45:45,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:47,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:49,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:51,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:45:51,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:52,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/config.yaml +2024-04-25 13:45:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:45:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:45:52,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:45:53,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:55,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:57,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:45:57,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:45:59,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:01,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:03,530 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:03,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:05,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:07,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:07,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:46:07,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:46:07,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:46:08,838 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:09,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:11,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:13,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:46:13,191 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:46:13,191 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:46:13,193 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:46:13,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:13,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:46:14,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:14,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:17,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:18,628 DEBUG SystemMonitor:156911 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-25 13:46:18,631 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:46:19,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:19,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:21,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:22,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:46:22,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:46:22,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:46:23,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:25,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:25,723 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:27,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:29,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:31,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:31,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:33,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:35,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:37,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:37,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:46:37,708 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:46:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:46:39,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:41,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:42,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:43,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:45,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:47,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:48,633 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:46:48,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:49,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:51,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:52,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:46:52,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:46:52,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:46:53,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:54,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:46:55,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:57,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:46:59,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:00,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:01,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:47:01,103 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:47:01,103 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:47:01,105 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:47:01,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:01,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:47:02,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:05,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:05,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:07,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:07,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:47:07,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:47:07,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:47:09,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:11,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:11,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:13,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:15,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:17,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:17,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:18,635 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:47:19,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:21,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:22,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:47:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:47:22,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:47:22,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:23,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:25,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:27,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:28,740 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:29,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:31,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:33,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:34,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:35,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:37,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:37,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:47:37,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:47:37,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:47:39,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:40,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:41,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:43,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:45,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:45,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:47,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:48,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:47:48,550 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:47:48,550 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:47:48,551 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:47:48,636 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:47:48,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:47:49,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:50,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:51,408 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:52,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:47:52,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:47:52,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:47:53,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:55,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:57,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:47:57,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:47:59,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:01,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:02,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:03,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:05,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:07,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:07,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:48:07,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:48:07,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:48:07,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:09,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:11,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:13,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:13,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:15,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:17,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:18,639 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:48:18,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:19,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:21,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:48:22,709 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:48:22,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:48:23,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:23,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:25,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:27,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:29,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:29,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:31,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:33,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:34,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:35,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:48:35,705 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:48:35,705 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:48:35,707 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:48:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:35,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:48:37,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:48:37,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:48:37,710 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:48:37,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:38,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:40,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:41,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:43,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:45,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:46,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:47,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:48,640 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:48:49,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:51,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:51,747 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:52,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:48:52,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:48:52,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:48:53,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:55,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:57,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:48:57,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:48:59,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:01,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:03,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:03,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:05,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:07,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:07,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:49:07,793 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:49:07,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:49:08,940 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:09,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:11,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:13,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:14,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:15,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:17,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:18,642 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:49:19,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:19,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:21,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:22,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:49:22,563 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:49:22,563 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:49:22,565 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:49:22,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:49:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:49:22,773 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:49:22,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:49:23,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:24,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:25,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:27,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:29,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:30,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:31,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:33,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:35,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:36,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:37,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:37,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:49:37,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:49:37,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:49:39,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:41,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:42,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:43,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:45,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:47,777 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:47,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:48,644 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:49:49,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:51,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:52,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:49:52,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:49:52,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:49:52,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:53,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:55,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:57,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:49:58,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:49:59,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:01,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:03,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:04,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:05,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:07,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:50:07,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:50:07,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:50:07,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:09,101 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:50:09,103 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:50:09,103 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:50:09,105 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:50:09,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:09,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:50:10,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:10,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:13,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:15,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:15,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:17,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:18,646 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:50:19,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:21,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:21,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:50:22,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:50:22,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:50:23,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:25,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:26,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:27,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:29,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:31,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:32,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:33,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:35,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:37,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:50:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:37,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:50:37,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:50:37,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:39,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:41,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:43,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:43,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:45,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:47,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:48,648 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:50:48,649 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:49,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:51,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:52,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:50:52,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:50:52,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:50:53,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:54,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:50:55,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:50:55,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:50:55,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:50:55,393 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:50:55,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:55,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:50:56,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:50:59,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:00,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:01,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:03,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:05,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:05,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:07,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:51:07,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:51:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:51:07,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:09,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:11,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:11,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:13,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:15,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:16,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:17,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:18,650 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:51:19,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:21,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:22,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:51:22,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:51:22,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:51:23,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:25,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:27,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:27,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:29,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:31,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:33,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:33,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:35,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:51:37,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:51:37,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:51:37,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:38,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:39,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:41,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:51:41,554 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:51:41,555 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:51:41,556 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:51:41,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:41,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:51:42,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:44,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:45,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:47,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:48,663 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:51:49,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:49,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:51,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:52,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:51:52,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:51:52,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:51:53,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:55,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:51:55,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:57,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:51:59,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:00,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:01,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:03,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:05,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:06,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:07,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:52:07,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:52:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:52:07,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:09,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:11,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:11,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:13,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:15,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:17,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:17,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:18,654 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:52:19,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:21,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:22,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:52:22,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:22,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:52:22,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:52:23,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:25,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:27,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:52:27,554 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:52:27,555 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:52:27,556 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:52:27,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:27,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:52:28,557 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:28,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:31,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:33,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:34,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:35,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:37,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:52:37,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:52:37,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:52:37,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:39,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:39,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:41,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:43,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:45,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:45,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:47,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:48,656 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:52:49,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:50,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:51,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:52,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:52:52,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:52:52,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:52:53,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:55,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:56,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:52:57,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:52:59,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:01,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:01,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:03,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:04,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:06,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:07,011 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:07,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:53:07,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:53:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:53:08,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:10,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:12,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:12,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:13,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:53:13,350 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:53:13,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:53:13,352 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:53:13,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:53:14,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:16,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:17,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:18,658 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:53:18,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:20,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:22,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:53:22,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:53:22,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:53:22,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:23,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:24,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:26,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:28,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:29,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:31,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:33,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:34,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:35,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:37,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:37,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:53:37,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:53:37,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:53:39,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:39,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:41,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:43,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:45,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:45,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:47,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:48,715 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:53:49,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:51,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:51,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:52,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:53:52,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:53:52,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:53:53,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:55,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:56,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:53:57,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:59,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:53:59,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:53:59,320 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:53:59,320 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:53:59,322 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:54:00,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:54:01,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:02,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:03,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:05,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:07,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:07,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:07,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:54:07,775 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:54:07,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:54:09,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:11,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:13,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:13,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:15,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:17,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:18,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:18,662 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:54:19,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:21,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:22,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:54:22,776 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:54:22,781 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:54:23,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:24,054 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:25,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:27,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:29,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:29,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:31,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:33,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:35,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:35,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:37,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:37,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:54:37,847 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:54:37,847 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:54:39,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:40,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:41,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:43,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:45,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:45,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:54:45,108 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:54:45,108 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:54:45,110 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:54:46,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:54:46,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:47,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:48,664 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:54:49,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:51,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:51,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:52,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:54:52,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:54:52,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:54:53,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:55,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:57,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:54:57,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:54:59,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:01,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:02,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:03,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:05,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:07,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:07,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:55:07,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:55:07,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:07,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:55:09,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:11,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:13,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:13,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:15,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:17,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:18,667 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:55:18,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:19,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:21,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:22,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:55:22,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:55:22,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:55:23,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:24,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:25,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:27,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:29,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:30,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:30,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:55:30,954 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:55:30,954 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:55:30,957 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:55:31,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:31,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:55:32,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:33,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:35,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:35,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:37,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:37,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:55:37,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:55:37,826 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:55:39,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:41,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:41,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:43,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:45,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:46,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:47,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:48,684 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:55:49,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:51,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:51,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:52,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:55:52,881 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:55:52,881 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:55:53,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:55,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:57,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:55:57,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:55:59,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:01,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:02,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:03,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:05,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:07,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:07,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:56:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:56:07,866 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:56:07,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:09,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:11,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:13,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:13,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:15,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:16,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:56:16,445 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:56:16,446 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:56:16,448 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:56:17,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:17,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:56:18,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:18,672 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:56:19,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:19,673 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:21,423 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:22,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:56:22,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:56:22,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:56:23,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:24,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:25,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:27,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:29,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:30,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:31,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:33,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:35,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:35,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:37,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:37,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:56:37,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:56:37,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:56:39,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:41,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:41,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:43,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:45,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:46,597 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:47,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:48,674 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:56:49,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:51,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:52,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:52,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:56:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:56:52,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:56:53,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:55,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:57,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:56:57,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:56:59,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:01,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:01,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:57:01,995 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:57:01,996 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:57:01,998 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:57:02,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:57:02,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:03,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:04,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:05,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:07,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:07,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:57:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:57:07,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:57:08,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:09,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:11,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:13,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:14,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:15,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:17,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:18,676 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:57:19,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:20,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:21,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:22,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:57:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:57:22,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:57:23,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:25,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:25,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:27,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:29,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:31,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:31,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:33,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:35,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:36,553 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:37,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:37,860 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:57:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:57:37,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:57:39,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:41,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:42,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:43,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:45,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:47,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:47,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:57:47,507 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:57:47,507 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:57:47,508 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:57:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:47,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:57:48,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:48,677 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:57:49,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:51,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:57:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:57:52,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:57:53,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:55,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:57,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:57:58,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:57:59,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:01,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:03,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:03,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:05,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:07,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:07,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:58:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:58:07,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:58:09,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:09,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:11,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:13,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:14,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:15,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:17,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:18,680 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:58:19,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:20,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:21,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:22,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:58:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:58:22,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:58:23,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:25,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:25,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:27,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:29,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:31,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:31,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:32,982 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:58:32,984 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:58:32,984 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:58:32,986 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:58:33,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:33,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:58:34,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:35,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:36,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:37,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:37,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:58:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:58:37,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:58:39,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:41,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:42,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:43,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:45,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:47,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:47,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:48,682 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:58:49,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:51,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:58:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:58:52,867 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:58:53,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:55,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:57,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:58:58,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:58:59,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:01,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:03,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:03,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:05,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:07,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:07,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:59:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:59:07,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:59:08,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:09,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:11,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:13,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:14,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:15,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:17,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:18,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 13:59:18,498 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 13:59:18,498 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 13:59:18,499 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 13:59:18,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 13:59:18,683 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:59:19,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:19,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:20,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:21,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:22,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:59:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:59:22,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:59:23,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:24,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:25,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:27,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:29,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:30,366 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:31,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:33,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:35,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:35,812 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:37,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:37,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:59:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:59:37,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:59:39,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:41,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:41,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:43,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:45,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:46,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:47,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:48,686 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 13:59:49,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:51,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:52,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:52,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 13:59:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 13:59:52,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 13:59:53,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:55,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:57,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 13:59:57,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 13:59:59,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:01,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:03,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:03,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:03,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:00:03,924 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:00:03,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:00:03,925 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:00:04,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:00:05,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:06,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:07,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:07,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:00:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:00:07,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:00:09,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:09,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:11,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:13,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:14,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:15,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:17,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:18,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:00:19,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:20,180 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:21,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:22,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:00:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:00:22,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:00:23,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:25,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:25,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:27,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:29,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:31,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:31,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:33,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:35,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:36,469 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:37,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:37,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:00:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:00:37,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:00:39,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:41,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:42,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:43,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:45,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:47,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:47,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:48,690 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:00:49,403 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:00:49,404 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:00:49,405 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:00:49,406 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:00:49,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:49,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:00:50,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:51,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:00:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:00:52,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:00:53,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:55,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:57,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:00:58,552 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:00:59,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:01,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:03,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:04,015 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:05,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:07,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:07,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:01:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:01:07,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:01:09,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:09,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:11,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:13,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:14,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:15,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:17,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:18,692 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:01:19,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:19,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:21,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:22,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:01:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:01:22,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:01:23,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:25,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:25,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:27,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:29,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:30,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:31,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:33,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:35,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:01:35,076 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:01:35,076 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:01:35,078 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:01:35,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:35,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:01:36,078 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:36,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:37,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:37,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:01:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:01:37,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:01:39,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:41,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:41,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:43,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:45,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:47,002 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:47,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:48,694 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:01:49,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:51,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:52,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:52,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:01:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:01:52,868 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:01:53,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:55,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:57,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:01:57,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:01:59,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:01,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:03,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:03,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:05,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:07,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:07,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:02:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:02:07,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:02:09,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:09,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:11,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:13,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:14,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:15,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:17,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:18,793 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:02:19,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:19,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:20,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:02:20,646 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:02:20,646 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:02:20,648 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:02:20,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:02:21,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:22,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:22,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:02:22,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:02:22,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:02:23,787 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:25,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:25,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:27,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:29,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:30,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:31,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:33,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:35,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:36,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:37,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:37,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:02:37,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:02:37,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:02:39,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:41,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:41,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:43,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:45,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:47,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:47,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:48,698 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:02:49,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:51,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:52,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:52,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:02:52,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:02:52,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:02:53,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:55,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:57,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:02:58,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:02:59,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:01,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:03,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:03,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:05,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:06,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:03:06,206 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:03:06,206 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:03:06,208 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:03:06,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:03:07,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:07,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:03:07,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:03:07,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:03:08,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:09,070 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:09,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:11,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:13,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:14,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:15,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:17,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:18,700 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:03:19,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:19,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:21,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:22,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:03:22,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:03:22,924 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:03:23,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:25,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:25,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:27,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:29,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:30,766 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:31,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:33,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:35,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:36,210 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:37,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:37,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:03:37,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:03:37,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:03:39,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:41,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:41,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:43,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:45,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:47,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:47,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:48,702 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:03:49,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:51,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:03:51,683 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:03:51,683 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:03:51,684 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:03:51,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:51,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:03:52,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:52,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:03:52,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:52,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:03:52,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:03:53,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:55,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:57,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:03:58,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:03:59,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:01,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:03,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:03,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:05,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:07,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:04:07,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:04:07,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:04:09,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:09,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:11,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:13,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:14,594 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:15,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:17,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:18,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:04:19,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:20,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:21,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:22,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:04:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:04:22,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:04:23,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:25,565 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:25,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:27,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:29,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:31,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:31,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:33,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:35,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:36,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:37,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:04:37,391 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:04:37,392 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:04:37,393 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:04:37,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:04:37,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:04:37,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:04:37,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:37,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:04:38,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:39,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:41,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:42,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:43,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:45,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:47,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:47,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:48,707 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:04:49,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:51,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:52,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:04:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:52,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:04:52,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:04:53,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:55,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:57,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:04:58,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:04:59,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:01,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:03,889 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:03,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:05,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:05:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:05:07,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:05:07,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:09,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:09,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:11,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:13,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:14,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:16,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:18,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:18,709 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:05:20,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:20,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:22,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:05:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:05:22,925 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:05:23,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:05:23,056 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:05:23,056 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:05:23,058 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:05:23,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:05:24,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:25,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:25,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:26,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:28,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:30,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:31,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:32,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:34,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:35,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:37,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:37,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:37,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:05:37,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:05:37,964 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:05:39,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:41,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:42,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:43,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:45,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:47,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:47,949 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:48,712 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:05:49,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:51,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:52,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:05:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:05:52,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:05:53,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:53,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:55,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:57,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:05:58,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:05:59,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:01,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:03,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:04,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:05,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:07,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:07,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:06:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:06:07,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:06:08,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:06:08,814 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:06:08,815 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:06:08,817 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:06:09,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:09,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:06:09,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:11,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:13,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:15,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:15,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:17,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:18,727 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:06:19,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:20,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:21,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:22,866 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:06:22,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:06:22,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:06:23,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:25,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:26,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:27,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:29,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:31,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:31,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:33,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:35,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:37,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:37,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:06:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:06:37,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:06:39,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:41,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:42,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:43,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:45,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:47,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:48,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:48,716 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:06:49,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:51,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:06:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:06:52,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:06:53,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:54,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:54,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:06:54,491 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:06:54,491 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:06:54,493 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:06:55,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:55,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:06:56,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:57,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:06:59,115 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:06:59,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:01,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:03,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:04,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:05,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:07,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:07:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:07:07,945 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:07:09,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:10,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:11,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:13,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:15,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:15,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:17,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:18,718 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:07:19,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:20,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:21,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:07:22,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:07:22,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:07:23,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:25,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:26,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:27,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:29,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:31,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:31,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:33,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:35,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:37,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:37,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:07:37,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:07:37,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:07:39,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:39,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:07:39,966 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:07:39,967 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:07:39,968 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:07:40,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:07:41,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:42,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:42,751 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:43,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:45,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:47,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:48,212 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:48,720 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:07:49,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:51,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:07:52,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:07:52,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:07:53,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:54,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:55,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:57,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:07:59,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:07:59,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:01,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:03,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:04,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:05,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:07,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:08:07,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:08:07,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:08:09,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:10,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:11,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:13,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:15,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:15,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:17,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:18,722 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:08:19,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:20,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:21,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:22,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:08:22,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:08:22,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:08:23,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:25,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:25,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:08:25,442 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:08:25,443 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:08:25,445 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:08:26,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:26,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:08:26,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:27,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:29,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:31,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:31,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:33,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:35,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:37,274 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:37,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:37,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:08:37,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:08:37,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:08:39,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:41,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:42,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:43,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:45,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:47,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:48,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:48,758 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:08:49,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:51,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:52,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:08:52,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:08:52,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:08:53,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:54,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:08:55,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:57,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:59,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:08:59,813 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:01,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:03,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:05,231 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:05,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:07,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:07,867 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:09:07,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:09:07,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:09:09,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:10,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:10,693 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:09:10,695 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:09:10,695 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:09:10,697 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:09:11,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:11,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:09:12,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:13,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:15,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:16,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:17,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:18,760 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:09:19,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:21,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:21,567 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:09:22,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:09:22,946 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:09:23,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:25,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:27,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:27,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:29,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:31,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:32,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:33,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:35,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:37,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:09:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:37,977 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:09:37,977 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:09:39,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:41,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:43,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:43,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:45,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:47,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:48,762 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:09:48,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:49,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:51,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:09:52,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:09:52,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:09:53,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:54,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:09:55,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:56,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:09:56,048 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:09:56,048 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:09:56,050 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:09:56,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:09:57,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:58,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:59,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:09:59,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:01,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:03,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:05,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:05,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:07,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:10:07,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:10:07,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:10:09,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:10,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:11,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:13,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:15,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:15,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:17,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:18,764 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:10:19,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:21,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:21,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:10:22,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:10:22,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:10:23,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:25,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:26,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:27,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:29,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:31,460 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:32,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:33,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:35,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:37,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:37,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:10:37,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:10:37,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:10:39,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:41,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:10:41,235 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:10:41,235 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:10:41,237 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:10:41,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:41,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:10:42,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:43,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:43,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:45,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:47,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:48,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:48,778 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:10:49,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:51,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:52,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:10:52,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:10:52,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:10:53,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:53,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:55,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:57,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:10:59,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:10:59,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:01,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:03,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:04,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:05,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:07,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:07,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:11:07,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:11:07,947 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:11:09,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:10,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:11,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:13,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:15,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:15,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:17,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:18,768 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:11:19,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:21,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:21,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:22,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:11:22,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:11:22,965 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:11:23,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:25,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:26,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:26,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:11:26,514 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:11:26,515 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:11:26,516 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:11:26,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:11:27,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:28,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:29,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:31,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:31,896 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:33,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:35,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:37,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:37,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:37,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:11:37,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:11:37,958 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:11:39,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:41,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:42,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:43,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:45,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:47,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:48,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:48,770 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:11:49,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:51,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:52,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:11:52,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:11:52,958 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:11:53,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:53,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:55,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:57,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:11:59,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:11:59,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:01,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:03,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:04,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:05,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:07,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:07,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:12:07,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:12:07,958 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:12:09,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:09,862 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:11,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:11,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:12:11,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:12:11,691 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:12:11,693 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:12:12,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:12,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:12:13,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:15,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:15,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:17,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:18,773 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:12:19,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:20,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:21,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:22,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:12:22,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:12:22,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:12:23,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:25,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:26,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:27,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:29,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:31,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:31,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:33,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:35,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:36,997 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:37,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:12:37,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:12:37,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:12:39,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:41,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:42,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:43,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:45,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:47,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:47,970 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:48,787 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:12:49,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:51,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:52,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:12:52,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:12:52,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:12:53,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:53,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:55,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:57,062 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:12:57,063 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:12:57,064 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:12:57,065 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:12:57,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:57,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:12:58,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:12:58,837 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:12:59,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:01,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:03,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:04,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:05,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:07,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:07,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:13:07,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:13:07,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:13:09,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:09,655 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:11,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:13,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:15,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:15,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:17,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:18,776 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:13:19,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:20,513 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:21,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:22,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:13:22,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:13:22,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:13:23,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:25,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:25,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:27,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:29,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:31,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:31,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:33,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:35,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:36,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:37,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:37,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:13:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:13:37,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:13:39,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:41,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:42,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:42,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:13:42,253 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:13:42,254 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:13:42,255 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:13:42,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:13:43,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:44,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:45,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:47,641 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:47,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:48,779 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:13:49,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:51,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:52,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:13:52,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:52,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:13:52,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:13:53,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:55,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:57,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:13:58,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:13:59,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:01,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:03,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:03,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:05,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:07,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:07,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:14:07,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:14:07,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:14:09,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:09,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:11,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:13,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:14,708 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:15,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:17,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:18,781 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:14:19,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:19,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:21,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:22,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:14:22,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:14:22,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:14:23,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:25,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:25,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:27,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:14:27,380 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:14:27,380 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:14:27,382 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:14:27,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:27,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:14:28,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:29,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:30,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:31,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:33,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:35,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:36,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:37,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:14:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:14:37,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:14:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:41,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:41,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:43,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:45,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:47,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:47,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:48,807 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:14:49,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:51,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:52,573 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:52,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:14:52,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:14:52,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:14:53,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:55,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:57,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:14:57,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:14:59,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:01,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:03,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:03,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:05,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:07,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:07,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:15:07,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:15:07,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:15:08,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:09,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:11,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:12,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:15:12,390 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:15:12,391 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:15:12,393 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:15:12,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:15:13,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:14,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:14,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:15,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:17,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:18,785 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:15:19,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:19,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:21,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:22,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:15:22,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:15:22,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:15:23,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:25,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:25,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:27,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:29,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:30,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:31,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:33,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:35,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:35,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:37,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:15:37,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:37,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:15:37,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:15:39,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:41,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:41,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:43,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:45,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:46,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:47,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:48,787 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:15:49,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:51,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:52,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:52,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:15:52,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:15:52,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:15:53,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:55,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:57,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:15:57,637 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:15:57,639 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:15:57,639 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:15:57,640 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:15:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:57,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:15:58,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:15:59,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:01,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:03,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:03,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:05,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:07,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:16:07,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:16:07,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:16:07,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:08,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:09,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:11,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:13,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:13,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:15,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:17,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:18,790 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:16:19,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:19,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:22,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:22,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:16:22,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:16:22,993 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:16:24,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:24,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:26,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:28,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:30,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:30,233 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:32,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:34,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:35,696 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:36,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:37,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:16:37,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:16:37,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:16:38,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:40,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:41,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:42,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:43,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:16:43,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:16:43,024 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:16:43,027 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:16:44,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:44,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:16:45,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:46,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:46,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:48,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:48,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:16:50,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:52,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:52,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:52,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:16:52,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:16:52,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:16:54,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:56,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:16:57,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:16:58,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:00,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:02,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:02,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:04,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:06,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:07,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:17:07,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:17:07,960 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:17:08,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:08,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:10,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:12,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:13,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:14,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:16,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:18,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:18,794 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:17:19,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:20,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:22,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:22,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:17:23,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:17:23,020 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:17:24,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:24,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:26,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:28,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:28,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:17:28,389 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:17:28,390 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:17:28,392 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:17:29,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:17:30,067 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:30,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:31,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:32,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:34,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:35,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:36,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:37,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:17:38,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:17:38,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:17:38,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:40,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:41,073 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:42,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:44,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:46,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:46,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:48,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:48,796 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:17:50,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:52,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:52,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:52,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:17:53,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:17:53,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:17:54,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:56,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:17:57,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:17:58,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:00,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:02,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:02,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:04,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:06,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:07,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:18:08,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:18:08,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:18:08,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:08,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:10,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:12,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:13,782 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:13,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:18:13,815 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:18:13,815 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:18:13,817 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:18:14,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:14,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:18:15,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:16,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:18,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:18,798 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:18:18,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:20,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:22,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:22,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:18:23,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:18:23,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:18:24,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:24,143 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:26,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:28,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:29,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:30,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:32,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:34,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:34,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:36,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:37,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:18:38,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:18:38,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:18:38,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:40,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:40,191 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:42,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:44,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:45,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:46,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:48,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:48,800 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:18:50,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:51,076 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:52,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:52,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:18:53,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:18:53,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:18:54,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:56,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:56,490 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:18:58,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:18:59,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:18:59,146 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:18:59,147 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:18:59,148 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:18:59,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:19:00,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:01,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:01,935 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:02,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:04,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:06,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:07,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:07,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:19:08,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:19:08,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:19:08,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:10,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:12,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:12,760 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:14,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:16,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:18,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:18,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:18,803 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:19:20,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:22,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:22,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:19:23,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:19:23,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:19:24,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:24,178 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:26,180 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:28,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:29,106 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:30,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:32,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:34,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:34,572 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:36,190 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:37,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:19:38,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:19:38,008 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:19:38,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:40,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:40,194 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:42,196 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:44,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:44,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:19:44,462 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:19:44,462 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:19:44,464 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:19:45,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:19:45,465 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:46,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:47,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:48,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:48,805 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:19:50,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:50,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:52,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:52,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:19:53,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:19:53,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:19:54,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:56,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:19:56,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:19:58,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:00,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:01,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:02,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:04,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:06,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:07,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:07,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:20:08,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:20:08,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:20:08,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:10,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:12,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:12,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:14,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:16,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:17,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:18,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:18,806 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:20:20,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:22,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:22,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:20:23,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:20:23,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:20:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:24,237 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:26,239 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:28,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:28,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:29,662 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:20:29,663 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:20:29,663 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:20:29,665 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:20:30,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:30,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:20:31,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:32,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:34,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:34,256 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:36,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:37,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:20:38,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:20:38,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:20:38,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:39,675 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:40,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:42,256 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:44,258 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:45,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:46,260 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:48,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:48,809 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:20:50,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:50,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:52,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:52,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:20:53,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:20:53,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:20:54,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:55,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:20:56,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:20:58,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:00,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:01,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:02,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:04,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:06,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:06,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:07,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:21:08,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:21:08,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:21:08,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:10,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:12,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:12,285 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:14,287 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:14,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:21:14,833 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:21:14,833 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:21:14,835 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:21:15,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:21:16,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:17,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:17,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:18,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:18,811 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:21:20,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:22,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:21:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:23,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:21:23,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:21:24,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:26,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:28,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:28,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:30,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:32,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:33,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:34,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:36,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:37,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:21:38,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:21:38,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:21:38,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:39,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:40,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:42,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:44,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:44,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:46,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:48,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:48,814 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:21:50,141 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:50,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:52,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:52,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:21:53,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:21:53,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:21:54,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:55,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:21:56,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:21:58,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:00,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:22:00,024 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:22:00,025 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:22:00,027 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:22:00,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:00,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:22:01,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:02,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:03,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:04,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:06,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:06,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:07,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:22:08,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:22:08,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:22:08,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:10,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:11,831 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:12,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:14,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:16,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:17,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:18,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:18,817 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:22:20,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:22,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:22,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:22:23,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:22:23,009 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:22:24,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:26,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:28,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:28,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:30,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:32,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:33,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:34,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:36,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:37,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:22:38,045 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:22:38,046 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:22:38,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:39,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:40,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:42,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:44,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:44,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:45,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:22:45,221 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:22:45,222 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:22:45,223 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:22:45,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:22:46,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:48,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:48,819 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:22:49,820 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:50,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:52,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:52,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:22:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:22:53,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:22:54,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:55,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:22:56,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:22:58,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:00,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:00,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:02,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:04,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:06,043 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:06,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:07,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:23:08,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:23:08,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:23:08,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:10,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:11,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:12,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:14,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:16,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:16,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:18,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:18,834 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:23:20,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:22,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:22,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:22,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:23:23,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:23:23,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:23:24,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:26,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:27,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:28,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:30,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:23:30,386 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:23:30,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:23:30,388 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:23:30,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:30,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:23:32,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:33,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:34,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:36,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:37,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:23:38,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:23:38,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:23:38,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:39,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:40,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:42,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:44,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:44,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:46,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:48,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:48,823 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:23:49,824 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:50,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:52,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:23:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:23:53,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:23:54,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:55,543 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:23:56,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:23:58,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:00,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:00,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:02,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:04,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:06,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:06,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:07,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:24:08,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:24:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:24:08,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:10,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:11,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:12,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:14,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:15,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:24:15,486 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:24:15,486 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:24:15,487 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:24:16,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:16,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:24:17,261 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:18,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:18,825 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:24:20,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:22,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:22,661 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:22,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:24:23,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:24:23,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:24:24,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:26,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:27,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:28,077 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:29,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:31,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:33,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:33,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:35,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:37,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:37,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:24:38,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:24:38,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:24:38,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:39,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:41,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:43,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:44,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:45,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:47,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:48,828 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:24:49,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:49,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:51,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:24:53,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:24:53,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:24:53,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:55,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:24:55,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:57,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:24:59,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:00,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:00,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:25:00,878 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:25:00,879 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:25:00,880 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:25:01,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:01,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:25:02,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:03,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:05,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:06,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:07,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:07,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:25:08,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:25:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:25:09,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:11,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:11,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:13,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:15,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:17,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:17,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:18,842 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:25:19,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:21,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:22,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:22,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:25:23,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:25:23,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:25:23,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:25,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:27,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:27,975 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:29,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:31,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:33,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:33,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:35,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:37,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:37,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:25:38,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:25:38,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:25:38,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:39,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:41,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:43,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:44,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:45,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:46,202 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:25:46,203 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:25:46,204 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:25:46,205 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:25:46,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:25:47,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:48,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:48,834 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:25:49,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:49,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:51,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:52,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:25:53,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:25:53,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:25:53,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:55,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:25:55,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:57,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:25:59,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:00,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:01,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:03,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:05,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:06,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:07,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:07,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:26:08,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:26:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:26:09,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:11,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:11,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:13,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:15,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:16,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:17,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:18,836 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:26:19,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:21,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:22,327 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:26:23,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:26:23,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:26:23,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:25,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:27,774 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:27,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:29,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:31,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:26:31,438 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:26:31,438 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:26:31,440 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:26:31,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:31,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:26:32,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:33,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:33,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:35,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:37,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:37,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:26:38,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:26:38,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:26:38,658 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:39,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:41,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:43,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:44,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:45,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:47,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:48,838 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:26:49,512 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:49,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:51,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:52,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:26:53,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:26:53,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:26:53,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:54,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:26:55,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:57,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:26:59,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:00,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:01,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:03,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:05,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:05,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:07,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:07,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:27:08,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:27:08,034 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:27:09,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:11,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:11,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:13,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:15,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:16,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:16,583 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:27:16,584 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:27:16,585 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:27:16,588 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:27:16,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:27:17,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:18,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:18,859 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:27:19,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:21,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:21,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:27:23,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:27:23,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:27:23,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:25,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:27,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:27,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:29,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:31,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:32,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:33,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:35,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:37,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:37,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:27:38,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:27:38,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:27:38,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:39,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:41,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:43,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:43,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:45,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:47,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:48,842 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:27:48,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:49,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:51,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:52,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:27:53,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:27:53,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:27:53,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:54,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:55,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:57,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:27:59,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:27:59,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:01,795 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:28:01,797 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:28:01,797 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:28:01,799 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:28:01,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:01,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:28:02,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:03,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:04,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:05,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:07,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:28:07,895 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:08,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:28:08,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:28:09,897 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:10,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:11,899 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:13,901 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:15,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:15,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:17,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:18,844 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:28:19,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:20,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:21,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:28:23,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:28:23,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:28:23,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:25,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:26,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:27,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:29,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:31,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:31,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:33,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:35,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:37,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:37,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:28:37,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:38,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:28:38,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:28:39,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:41,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:42,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:43,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:45,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:46,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:28:46,968 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:28:46,968 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:28:46,970 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:28:47,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:47,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:28:47,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:48,846 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:28:48,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:49,938 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:51,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:52,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:28:53,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:28:53,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:28:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:53,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:55,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:57,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:28:58,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:28:59,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:01,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:03,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:04,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:05,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:07,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:29:07,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:08,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:29:08,035 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:29:09,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:09,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:11,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:13,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:14,987 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:15,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:17,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:18,849 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:29:19,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:20,396 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:21,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:22,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:29:23,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:29:23,107 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:29:23,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:25,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:25,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:27,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:29,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:31,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:31,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:32,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:29:32,056 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:29:32,057 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:29:32,058 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:29:32,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:29:33,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:34,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:36,642 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:37,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:29:37,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:38,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:29:38,087 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:29:39,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:41,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:42,040 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:43,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:45,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:47,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:47,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:48,851 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:29:49,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:51,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:52,840 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:29:52,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:29:53,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:29:53,087 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:29:54,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:56,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:58,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:29:58,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:00,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:02,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:03,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:04,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:06,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:07,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:30:08,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:08,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:30:08,161 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:30:09,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:10,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:12,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:14,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:14,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:16,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:17,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:30:17,109 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:30:17,110 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:30:17,111 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:30:18,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:18,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:30:18,853 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:30:19,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:19,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:20,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:22,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:22,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:30:23,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:30:23,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:30:24,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:25,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:26,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:28,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:30,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:30,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:32,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:34,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:36,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:36,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:37,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:30:38,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:38,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:30:38,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:30:40,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:41,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:42,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:44,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:46,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:46,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:48,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:48,855 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:30:50,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:52,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:52,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:52,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:30:53,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:30:53,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:30:54,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:56,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:30:57,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:30:58,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:00,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:02,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:02,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:31:02,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:31:02,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:31:02,249 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:31:03,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:31:03,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:04,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:05,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:06,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:07,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:31:08,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:08,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:31:08,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:31:09,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:10,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:12,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:14,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:14,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:16,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:18,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:18,857 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:31:20,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:20,301 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:22,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:22,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:31:23,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:31:23,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:31:24,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:25,704 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:26,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:28,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:30,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:31,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:32,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:34,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:36,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:36,502 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:37,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:31:38,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:31:38,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:31:38,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:40,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:41,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:42,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:44,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:46,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:47,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:47,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:31:47,393 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:31:47,393 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:31:47,395 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:31:48,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:48,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:31:48,859 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:31:49,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:50,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:52,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:52,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:31:52,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:31:53,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:31:53,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:31:54,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:56,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:58,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:31:58,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:00,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:02,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:03,568 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:04,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:06,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:07,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:32:08,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:32:08,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:32:08,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:08,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:10,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:12,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:14,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:14,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:16,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:18,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:18,862 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:32:19,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:20,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:22,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:22,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:32:23,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:32:23,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:32:24,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:25,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:26,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:28,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:30,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:30,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:32,153 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:32,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:32:32,442 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:32:32,442 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:32:32,443 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:32:33,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:32:34,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:35,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:36,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:36,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:37,913 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:32:38,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:32:38,088 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:32:38,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:40,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:41,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:42,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:44,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:46,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:46,834 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:48,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:48,864 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:32:50,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:52,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:52,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:52,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:32:53,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:32:53,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:32:54,175 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:56,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:32:57,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:32:58,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:00,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:02,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:03,094 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:04,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:06,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:07,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:33:08,088 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:33:08,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:33:08,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:08,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:10,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:12,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:13,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:14,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:16,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:17,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:33:17,543 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:33:17,544 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:33:17,545 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:33:18,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:18,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:33:18,866 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:33:19,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:19,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:20,201 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:22,203 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:22,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:33:23,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:33:23,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:33:24,205 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:24,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:26,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:28,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:30,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:30,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:32,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:34,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:35,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:36,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:37,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:33:38,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:33:38,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:33:38,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:40,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:40,941 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:42,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:44,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:46,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:46,377 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:48,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:48,897 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:33:50,230 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:51,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:52,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:52,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:33:53,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:33:53,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:33:54,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:56,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:33:57,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:33:58,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:00,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:02,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:02,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:02,715 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:34:02,716 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:34:02,716 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:34:02,718 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:34:03,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:34:04,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:05,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:06,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:07,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:07,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:34:08,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:34:08,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:34:08,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:10,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:12,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:13,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:14,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:16,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:18,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:18,870 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:34:18,871 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:20,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:22,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:22,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:34:23,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:34:23,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:34:24,211 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:24,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:26,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:28,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:29,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:30,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:32,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:34,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:35,300 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:36,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:37,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:34:38,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:34:38,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:34:38,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:40,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:40,716 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:42,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:44,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:46,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:46,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:48,001 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:34:48,002 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:34:48,002 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:34:48,004 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:34:48,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:48,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:34:48,872 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:34:49,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:50,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:51,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:52,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:52,900 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:34:53,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:34:53,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:34:54,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:56,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:34:56,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:34:58,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:00,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:02,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:02,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:04,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:06,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:07,818 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:07,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:35:08,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:35:08,089 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:35:08,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:10,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:12,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:13,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:14,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:16,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:18,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:18,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:18,874 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:35:20,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:22,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:35:23,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:35:23,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:35:24,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:24,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:26,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:28,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:29,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:30,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:32,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:33,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:35:33,297 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:35:33,298 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:35:33,299 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:35:33,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:35:34,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:35,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:36,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:37,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:35:38,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:35:38,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:35:38,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:40,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:40,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:42,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:44,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:45,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:46,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:48,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:48,889 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:35:50,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:51,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:52,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:52,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:35:53,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:35:53,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:35:54,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:56,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:35:56,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:35:58,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:00,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:02,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:02,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:04,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:06,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:07,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:36:08,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:36:08,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:36:08,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:10,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:12,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:13,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:14,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:16,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:18,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:18,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:18,714 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:36:18,715 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:36:18,716 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:36:18,717 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:36:18,878 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:36:19,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:36:20,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:22,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:22,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:36:23,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:36:23,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:36:24,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:24,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:26,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:28,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:29,547 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:30,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:32,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:34,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:34,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:36,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:37,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:36:38,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:36:38,090 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:36:38,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:40,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:40,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:42,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:44,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:45,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:46,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:48,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:48,880 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:36:50,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:51,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:52,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:52,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:36:53,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:36:53,161 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:36:54,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:56,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:36:56,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:36:58,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:00,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:02,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:02,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:03,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:37:03,970 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:37:03,970 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:37:03,972 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:37:04,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:04,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:37:06,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:07,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:07,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:37:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:37:08,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:37:08,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:10,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:12,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:12,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:14,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:16,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:18,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:18,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:18,883 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:37:20,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:22,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:37:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:37:23,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:37:23,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:24,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:26,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:28,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:29,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:30,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:32,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:34,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:34,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:36,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:37,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:37:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:37:38,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:37:38,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:40,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:40,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:42,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:44,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:45,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:46,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:48,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:48,885 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:37:49,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:37:49,319 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:37:49,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:37:49,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:37:49,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:37:50,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:51,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:52,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:52,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:37:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:37:53,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:37:54,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:56,526 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:37:56,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:37:58,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:00,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:01,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:02,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:04,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:06,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:07,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:07,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:38:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:38:08,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:38:08,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:10,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:12,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:12,769 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:14,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:16,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:18,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:18,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:18,887 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:38:20,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:22,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:38:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:38:23,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:38:23,257 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:24,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:26,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:28,267 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:28,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:30,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:32,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:33,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:34,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:38:34,578 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:38:34,578 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:38:34,580 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:38:34,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:34,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:38:36,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:37,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:38:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:38:38,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:38:38,728 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:39,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:40,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:42,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:44,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:44,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:46,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:48,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:48,889 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:38:50,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:50,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:52,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:52,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:38:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:38:53,148 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:38:54,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:55,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:38:56,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:57,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:38:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:00,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:03,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:05,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:06,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:07,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:07,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:39:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:39:08,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:39:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:11,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:13,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:17,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:18,891 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:39:19,733 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:39:19,734 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:39:19,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:39:19,736 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:39:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:39:20,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:21,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:22,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:39:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:39:23,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:39:23,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:25,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:27,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:27,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:29,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:31,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:33,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:33,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:35,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:37,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:37,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:39:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:39:38,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:39:39,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:39,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:41,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:43,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:44,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:45,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:47,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:48,893 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:39:49,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:49,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:51,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:52,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:39:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:39:53,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:39:53,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:55,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:39:55,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:57,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:39:59,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:00,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:01,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:03,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:04,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:40:04,993 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:40:04,993 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:40:04,995 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:40:05,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:05,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:40:05,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:06,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:07,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:07,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:40:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:40:08,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:40:09,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:11,399 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:11,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:13,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:15,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:16,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:17,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:18,895 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:40:19,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:21,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:22,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:22,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:40:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:40:23,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:40:23,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:25,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:27,817 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:27,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:29,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:31,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:33,296 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:33,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:35,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:37,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:37,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:40:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:40:38,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:40:38,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:41,849 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:43,851 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:44,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:45,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:47,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:48,898 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:40:49,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:49,899 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:50,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:40:50,483 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:40:50,483 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:40:50,485 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:40:50,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:40:51,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:52,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:52,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:40:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:40:53,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:40:53,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:55,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:40:55,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:57,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:40:59,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:00,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:01,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:03,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:05,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:05,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:07,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:07,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:41:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:41:08,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:41:09,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:11,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:11,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:13,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:15,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:16,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:17,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:18,900 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:41:19,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:21,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:22,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:22,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:41:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:41:23,149 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:41:23,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:25,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:27,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:27,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:29,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:31,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:33,321 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:33,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:35,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:35,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:41:35,986 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:41:35,986 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:41:35,988 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:41:36,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:36,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:41:37,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:37,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:41:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:41:38,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:41:39,311 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:39,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:41,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:43,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:45,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:45,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:47,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:48,902 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:41:49,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:50,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:51,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:52,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:41:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:41:53,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:41:53,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:55,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:41:55,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:57,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:41:59,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:01,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:01,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:03,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:05,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:06,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:07,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:42:07,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:42:08,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:42:09,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:11,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:12,356 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:13,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:15,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:17,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:17,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:18,916 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:42:19,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:21,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:42:21,492 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:42:21,492 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:42:21,494 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:42:21,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:21,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:42:22,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:22,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:42:22,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:42:23,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:42:23,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:25,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:27,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:28,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:29,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:31,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:33,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:34,170 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:35,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:37,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:42:37,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:42:38,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:42:39,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:39,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:41,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:43,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:44,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:45,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:47,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:48,906 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:42:49,907 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:49,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:51,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:52,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:42:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:42:53,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:42:53,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:55,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:42:55,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:57,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:42:59,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:00,617 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:01,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:03,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:05,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:06,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:06,894 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:43:06,895 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:43:06,896 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:43:06,897 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:43:06,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:43:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:43:07,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:43:08,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:43:08,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:10,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:11,489 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:12,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:14,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:16,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:16,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:18,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:18,908 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:43:20,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:22,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:22,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:43:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:43:23,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:43:24,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:26,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:27,785 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:28,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:30,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:32,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:33,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:34,023 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:36,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:43:38,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:43:38,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:43:38,298 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:40,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:42,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:44,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:44,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:46,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:48,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:48,910 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:43:49,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:50,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:52,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:52,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:43:52,323 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:43:52,324 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:43:52,325 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:43:52,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:43:53,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:43:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:43:53,150 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:43:54,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:55,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:55,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:43:56,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:43:58,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:00,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:00,528 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:02,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:04,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:05,965 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:06,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:44:08,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:44:08,151 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:44:10,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:11,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:12,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:14,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:16,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:16,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:18,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:18,925 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:44:20,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:22,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:22,279 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:44:23,194 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:44:23,194 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:44:24,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:26,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:27,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:28,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:30,081 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:32,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:33,173 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:34,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:36,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:37,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:44:37,623 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:44:37,623 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:44:37,625 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:44:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:44:38,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:38,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:44:38,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:44:38,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:44:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:39,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:40,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:42,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:44,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:44,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:46,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:48,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:48,915 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:44:49,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:50,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:52,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:52,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:44:53,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:44:53,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:44:54,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:55,729 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:44:56,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:44:58,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:00,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:01,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:02,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:04,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:06,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:06,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:45:08,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:08,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:45:08,174 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:45:10,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:12,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:12,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:14,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:16,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:17,762 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:18,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:18,917 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:45:20,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:22,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:22,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:45:23,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:45:23,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:45:23,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:45:23,508 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:45:23,508 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:45:23,510 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:45:24,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:24,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:45:25,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:26,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:28,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:28,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:30,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:32,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:34,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:34,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:36,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:45:38,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:45:38,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:45:38,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:39,391 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:40,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:42,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:44,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:45,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:46,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:48,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:48,919 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:45:50,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:50,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:52,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:52,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:45:53,174 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:45:53,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:45:54,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:55,628 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:45:56,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:45:58,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:00,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:00,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:02,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:04,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:06,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:06,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:07,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:46:08,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:46:08,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:46:08,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:08,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:46:08,778 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:46:08,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:46:08,779 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:46:09,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:46:10,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:11,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:11,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:12,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:14,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:16,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:16,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:18,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:18,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:46:20,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:22,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:22,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:22,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:46:23,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:46:23,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:46:24,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:26,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:27,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:28,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:32,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:33,249 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:34,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:37,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:46:38,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:46:38,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:46:38,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:40,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:42,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:44,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:44,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:46,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:48,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:48,923 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:46:49,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:50,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:52,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:52,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:46:53,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:46:53,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:46:54,110 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:46:54,111 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:46:54,112 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:46:54,113 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:46:54,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:54,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:46:55,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:46:55,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:56,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:46:58,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:00,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:00,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:02,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:04,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:05,981 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:06,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:07,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:47:08,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:47:08,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:47:08,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:10,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:11,402 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:12,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:14,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:16,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:16,819 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:18,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:18,925 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:47:20,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:22,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:22,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:22,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:47:23,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:47:23,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:47:24,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:26,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:27,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:28,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:30,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:32,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:33,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:34,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:36,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:37,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:47:38,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:47:38,175 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:47:38,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:38,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:39,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:47:39,335 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:47:39,336 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:47:39,337 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:47:39,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:47:40,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:41,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:42,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:43,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:44,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:46,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:48,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:48,927 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:47:49,928 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:50,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:52,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:52,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:47:53,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:47:53,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:47:54,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:55,707 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:47:56,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:47:58,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:00,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:01,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:02,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:04,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:06,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:06,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:07,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:48:08,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:48:08,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:48:08,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:10,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:12,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:12,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:14,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:16,519 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:17,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:18,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:18,929 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:48:20,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:22,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:22,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:22,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:48:23,175 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:48:23,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:48:24,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:24,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:48:24,711 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:48:24,711 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:48:24,714 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:48:25,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:48:26,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:27,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:28,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:28,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:30,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:32,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:33,764 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:34,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:36,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:48:38,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:48:38,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:48:38,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:39,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:40,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:42,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:44,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:44,643 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:46,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:48,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:48,932 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:48:49,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:50,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:52,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:48:53,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:48:53,176 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:48:54,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:55,484 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:48:56,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:48:58,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:00,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:00,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:02,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:04,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:06,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:06,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:49:08,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:49:08,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:49:08,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:09,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:49:09,991 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:49:09,991 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:49:09,993 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:49:10,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:10,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:49:11,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:11,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:12,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:14,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:16,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:17,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:18,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:18,934 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:49:20,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:22,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:22,650 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:22,944 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:49:23,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:49:23,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:49:24,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:26,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:28,064 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:28,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:30,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:32,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:33,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:34,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:36,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:49:38,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:49:38,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:49:38,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:38,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:40,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:42,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:44,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:44,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:46,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:48,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:48,936 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:49:49,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:50,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:52,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:49:53,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:49:53,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:49:54,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:55,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:49:55,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:49:55,210 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:49:55,211 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:49:55,212 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:49:55,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:49:56,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:57,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:49:58,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:00,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:00,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:02,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:04,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:06,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:06,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:50:08,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:50:08,229 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:50:08,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:10,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:11,551 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:12,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:14,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:16,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:16,966 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:18,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:18,938 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:50:20,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:22,383 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:22,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:22,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:50:23,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:50:23,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:50:24,647 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:26,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:27,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:28,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:30,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:32,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:33,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:34,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:36,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:37,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:50:38,229 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:50:38,230 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:50:38,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:38,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:40,499 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:50:40,501 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:50:40,502 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:50:40,503 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:50:40,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:40,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:50:42,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:44,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:44,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:46,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:48,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:48,940 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:50:49,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:50,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:52,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:52,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:50:53,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:50:53,264 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:50:54,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:54,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:50:56,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:50:58,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:00,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:00,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:02,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:04,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:05,799 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:06,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:07,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:51:08,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:51:08,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:51:08,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:10,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:11,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:12,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:14,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:16,639 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:16,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:18,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:18,943 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:51:20,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:22,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:22,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:22,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:51:23,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:51:23,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:51:24,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:25,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:51:25,729 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:51:25,729 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:51:25,731 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:51:26,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:26,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:51:27,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:28,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:30,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:32,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:32,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:34,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:36,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:37,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:51:38,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:51:38,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:51:38,355 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:38,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:40,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:42,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:43,803 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:44,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:46,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:48,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:48,944 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:51:48,946 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:50,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:52,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:52,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:51:53,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:51:53,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:51:54,359 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:51:54,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:56,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:51:58,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:00,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:00,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:02,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:04,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:05,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:06,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:07,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:52:08,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:52:08,244 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:52:08,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:10,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:10,925 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:10,957 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:52:10,959 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:52:10,959 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:52:10,961 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:52:11,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:52:12,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:14,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:16,361 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:16,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:18,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:18,947 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:52:20,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:21,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:22,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:22,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:52:23,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:52:23,245 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:52:24,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:26,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:27,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:28,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:30,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:32,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:32,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:34,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:36,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:37,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:52:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:38,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:52:38,303 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:52:38,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:40,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:42,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:43,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:44,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:46,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:48,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:48,949 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:52:48,950 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:50,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:52,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:52,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:52:53,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:52:53,283 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:52:54,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:52:54,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:56,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:52:56,386 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:52:56,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:52:56,388 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:52:56,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:56,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:52:58,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:52:59,976 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:00,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:02,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:04,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:05,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:06,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:07,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:53:08,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:53:08,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:53:08,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:10,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:10,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:12,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:14,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:16,260 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:16,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:18,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:18,951 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:53:20,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:21,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:22,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:22,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:53:23,283 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:53:23,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:53:24,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:26,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:27,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:28,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:30,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:32,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:32,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:34,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:36,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:37,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:53:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:38,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:53:38,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:53:38,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:40,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:41,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:53:41,686 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:53:41,687 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:53:41,689 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:53:41,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:53:42,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:43,464 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:44,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:46,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:48,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:48,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:48,953 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:53:50,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:52,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:52,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:53:53,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:53:53,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:53:54,334 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:53:54,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:56,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:58,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:53:59,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:00,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:02,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:04,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:05,255 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:06,869 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:07,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:54:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:54:08,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:54:08,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:10,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:10,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:12,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:14,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:16,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:16,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:18,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:18,955 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:54:20,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:21,711 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:22,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:22,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:54:23,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:54:23,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:54:24,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:26,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:27,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:27,203 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:54:27,205 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:54:27,206 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:54:27,207 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:54:27,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:54:28,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:30,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:32,635 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:32,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:34,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:36,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:54:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:38,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:54:38,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:54:38,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:40,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:42,909 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:43,546 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:44,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:46,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:48,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:48,957 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:54:48,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:50,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:52,920 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:52,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:54:53,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:54:53,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:54:54,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:54:54,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:56,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:58,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:54:59,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:00,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:02,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:04,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:05,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:06,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:07,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:55:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:55:08,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:55:08,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:10,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:11,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:12,938 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:55:12,940 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:55:12,941 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:55:12,942 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:55:12,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:12,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:55:14,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:16,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:16,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:18,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:18,960 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:55:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:22,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:22,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:22,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:55:23,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:55:23,284 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:55:24,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:26,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:27,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:28,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:30,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:32,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:32,918 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:34,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:36,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:55:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:38,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:38,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:55:38,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:55:40,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:42,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:43,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:44,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:46,163 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:48,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:48,480 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:48,962 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:55:50,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:52,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:52,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:55:53,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:55:53,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:55:54,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:54,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:55:56,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:58,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:55:58,493 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:55:58,494 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:55:58,495 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:55:58,496 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:55:59,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:55:59,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:00,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:02,184 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:04,186 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:04,915 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:06,188 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:07,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:56:08,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:08,284 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:56:08,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:56:10,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:10,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:12,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:14,198 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:15,790 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:16,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:18,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:18,964 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:56:20,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:21,223 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:22,207 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:22,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:56:23,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:56:23,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:56:24,209 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:26,211 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:26,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:28,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:30,216 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:32,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:32,218 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:34,220 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:36,222 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:37,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:37,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:56:38,224 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:38,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:56:38,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:56:40,226 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:42,228 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:42,932 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:43,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:56:43,777 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:56:43,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:56:43,779 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:56:44,231 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:44,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:56:46,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:48,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:48,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:48,977 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:56:50,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:52,241 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:52,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:56:53,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:56:53,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:56:53,434 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:56:54,243 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:56,245 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:58,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:56:59,272 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:00,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:02,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:04,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:04,713 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:06,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:07,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:57:08,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:08,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:57:08,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:57:10,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:10,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:12,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:14,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:15,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:16,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:18,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:18,968 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:57:20,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:21,004 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:22,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:22,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:57:23,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:57:23,285 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:57:24,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:26,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:26,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:28,283 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:29,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:57:29,105 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:57:29,106 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:57:29,108 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:57:29,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:57:30,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:31,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:32,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:34,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:36,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:37,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:57:38,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:57:38,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:57:38,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:40,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:42,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:42,749 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:44,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:46,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:48,166 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:48,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:48,971 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:57:50,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:52,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:52,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:57:53,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:57:53,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:57:53,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:57:54,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:56,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:58,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:57:59,008 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:00,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:02,321 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:04,323 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:04,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:06,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:07,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:58:08,285 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:58:08,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:58:08,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:09,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:10,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:12,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:14,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:14,367 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:58:14,369 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:58:14,369 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:58:14,371 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:58:15,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:58:15,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:16,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:18,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:18,972 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:58:20,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:20,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:22,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:22,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:58:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:58:23,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:58:24,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:26,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:26,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:28,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:30,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:31,580 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:32,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:34,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:36,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:37,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:37,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:58:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:58:38,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:58:38,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:40,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:42,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:42,441 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:44,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:46,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:47,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:48,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:48,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:58:50,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:52,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:52,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:58:52,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:53,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:58:53,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:58:54,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:56,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:58,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:58:58,702 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:58:59,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:58:59,536 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:58:59,537 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:58:59,540 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:59:00,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:00,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:59:02,382 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:04,108 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:04,384 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:06,386 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:07,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:59:08,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:59:08,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:59:08,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:09,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:10,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:12,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:14,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:14,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:16,396 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:18,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:18,977 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:59:20,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:20,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:22,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:23,027 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:59:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:59:23,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:59:24,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:25,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:26,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:28,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:30,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:31,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:32,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:34,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:36,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:36,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:38,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:59:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:59:38,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:59:38,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:40,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:41,968 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:42,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:44,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:44,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 14:59:44,611 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 14:59:44,612 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 14:59:44,614 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 14:59:45,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 14:59:46,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:47,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:48,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:48,979 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 14:59:50,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:52,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:52,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:53,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 14:59:53,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 14:59:53,286 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 14:59:54,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:56,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 14:59:58,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 14:59:58,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:00,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:02,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:03,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:04,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:06,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:08,069 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:00:08,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:00:08,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:00:08,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:09,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:10,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:12,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:14,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:15,184 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:16,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:18,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:18,981 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:00:20,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:20,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:22,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:23,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:00:23,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:00:23,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:00:24,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:26,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:26,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:28,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:29,691 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:00:29,693 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:00:29,693 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:00:29,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:00:30,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:30,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:00:31,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:32,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:34,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:36,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:36,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:38,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:00:38,286 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:00:38,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:00:38,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:40,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:42,295 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:42,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:44,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:46,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:47,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:48,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:48,993 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:00:50,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:52,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:53,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:00:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:53,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:00:53,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:00:54,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:56,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:00:58,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:00:58,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:00,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:02,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:03,890 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:04,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:06,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:08,049 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:01:08,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:01:08,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:01:08,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:09,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:10,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:12,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:14,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:14,725 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:14,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:01:14,758 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:01:14,759 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:01:14,760 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:01:15,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:01:16,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:18,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:18,985 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:01:19,986 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:20,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:22,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:01:23,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:01:23,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:01:24,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:25,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:26,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:28,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:30,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:30,927 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:32,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:34,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:36,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:36,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:38,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:01:38,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:01:38,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:01:38,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:40,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:41,705 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:42,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:44,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:46,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:47,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:48,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:48,987 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:01:50,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:52,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:52,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:01:53,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:01:53,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:01:54,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:56,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:57,895 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:01:58,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:01:59,731 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:01:59,733 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:01:59,734 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:01:59,735 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:02:00,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:00,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:02:02,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:03,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:04,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:06,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:08,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:02:08,287 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:02:08,287 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:02:08,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:08,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:10,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:12,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:14,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:14,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:16,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:18,579 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:18,990 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:02:19,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:20,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:22,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:02:23,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:02:23,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:02:24,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:24,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:26,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:28,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:30,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:30,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:32,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:34,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:35,806 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:36,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:38,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:02:38,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:02:38,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:02:38,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:40,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:41,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:42,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:44,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:44,857 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:02:44,859 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:02:44,859 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:02:44,861 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:02:45,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:02:46,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:46,632 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:48,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:49,003 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:02:50,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:52,036 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:52,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:02:53,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:02:53,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:02:54,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:56,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:02:57,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:02:58,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:00,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:02,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:02,879 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:04,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:06,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:08,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:03:08,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:08,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:03:08,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:03:08,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:10,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:12,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:13,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:14,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:16,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:18,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:18,994 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:03:18,995 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:20,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:22,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:23,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:03:23,291 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:03:23,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:03:24,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:24,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:26,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:28,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:30,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:30,186 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:03:30,188 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:03:30,188 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:03:30,190 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:03:30,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:30,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:03:32,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:34,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:35,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:36,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:38,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:03:38,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:03:38,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:03:38,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:40,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:41,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:42,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:44,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:46,437 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:46,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:48,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:48,997 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:03:50,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:51,886 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:52,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:53,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:03:53,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:03:53,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:03:54,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:56,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:03:57,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:03:58,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:00,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:02,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:02,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:04,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:06,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:08,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:04:08,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:08,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:04:08,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:04:08,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:10,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:12,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:13,590 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:14,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:15,430 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:04:15,432 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:04:15,433 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:04:15,434 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:04:15,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:04:16,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:18,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:18,999 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:04:19,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:20,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:22,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:23,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:04:23,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:04:23,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:04:24,445 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:24,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:26,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:28,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:29,922 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:30,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:32,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:34,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:35,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:36,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:38,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:04:38,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:04:38,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:04:38,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:40,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:40,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:42,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:44,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:46,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:46,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:48,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:49,001 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:04:50,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:51,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:52,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:53,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:04:53,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:04:53,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:04:54,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:56,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:04:57,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:04:58,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:00,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:00,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:05:00,831 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:05:00,832 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:05:00,833 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:05:01,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:05:02,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:02,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:04,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:06,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:08,022 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:08,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:05:08,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:05:08,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:05:08,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:10,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:12,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:13,442 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:14,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:16,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:18,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:18,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:19,003 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:05:20,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:22,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:23,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:05:23,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:05:23,292 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:05:24,314 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:24,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:26,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:28,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:29,736 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:30,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:32,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:34,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:35,161 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:36,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:38,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:05:38,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:05:38,293 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:05:38,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:40,575 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:40,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:42,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:45,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:45,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:46,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:05:46,022 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:05:46,022 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:05:46,024 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:05:47,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:47,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:05:49,006 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:05:49,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:51,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:51,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:53,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:53,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:05:53,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:05:53,330 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:05:55,017 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:56,868 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:05:57,019 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:05:59,021 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:01,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:02,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:03,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:05,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:07,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:07,758 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:08,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:06:08,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:06:08,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:06:09,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:11,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:13,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:13,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:15,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:17,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:18,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:19,008 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:06:19,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:21,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:23,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:23,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:06:23,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:06:23,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:06:24,063 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:25,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:27,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:29,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:29,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:31,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:31,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:06:31,341 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:06:31,341 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:06:31,342 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:06:32,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:06:33,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:34,933 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:35,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:37,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:38,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:06:38,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:06:38,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:06:39,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:40,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:41,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:43,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:45,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:45,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:47,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:49,011 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:06:49,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:51,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:51,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:53,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:53,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:06:53,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:06:53,317 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:06:55,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:56,544 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:06:57,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:06:59,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:01,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:02,097 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:03,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:05,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:07,094 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:07,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:07:08,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:07:08,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:07:09,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:11,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:12,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:13,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:15,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:16,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:07:16,613 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:07:16,614 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:07:16,615 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:07:17,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:17,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:07:18,398 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:19,013 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:07:19,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:21,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:23,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:07:23,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:23,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:07:23,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:07:23,452 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:25,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:27,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:28,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:29,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:31,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:33,121 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:33,877 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:35,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:37,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:38,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:07:38,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:07:38,337 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:07:39,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:39,444 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:41,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:43,132 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:44,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:45,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:47,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:49,015 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:07:49,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:50,160 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:51,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:53,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:07:53,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:53,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:07:53,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:07:55,143 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:55,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:07:57,145 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:07:59,147 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:01,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:01,149 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:02,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:08:02,031 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:08:02,031 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:08:02,033 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:08:02,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:08:03,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:05,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:06,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:07,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:08:08,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:08:08,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:08:09,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:11,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:12,092 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:13,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:15,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:17,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:17,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:19,017 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:08:19,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:21,171 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:22,998 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:23,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:08:23,173 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:23,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:08:23,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:08:25,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:27,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:28,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:29,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:31,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:33,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:33,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:35,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:37,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:38,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:08:38,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:08:38,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:08:39,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:39,390 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:41,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:43,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:44,780 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:45,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:47,197 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:47,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:08:47,437 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:08:47,438 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:08:47,439 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:08:48,199 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:08:49,019 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:08:49,200 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:50,222 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:51,202 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:53,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:08:53,204 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:53,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:08:53,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:08:55,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:55,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:08:57,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:08:59,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:01,178 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:01,212 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:03,214 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:05,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:06,660 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:07,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:08,104 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:09:08,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:09:08,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:09:09,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:11,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:12,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:13,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:15,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:17,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:17,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:19,034 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:09:19,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:21,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:23,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:23,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:09:23,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:23,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:09:23,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:09:25,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:27,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:28,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:29,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:31,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:33,012 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:09:33,014 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:09:33,015 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:09:33,018 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:09:33,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:33,247 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:09:34,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:35,249 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:37,251 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:38,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:09:38,318 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:09:38,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:09:39,253 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:39,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:41,255 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:43,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:44,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:45,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:47,262 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:49,024 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:09:49,264 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:50,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:51,266 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:53,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:09:53,268 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:53,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:09:53,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:09:55,270 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:55,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:09:57,272 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:09:59,274 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:01,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:01,276 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:03,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:05,280 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:06,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:07,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:08,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:10:08,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:10:08,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:10:09,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:11,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:12,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:13,289 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:15,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:17,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:17,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:18,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:10:18,399 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:10:18,400 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:10:18,401 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:10:19,026 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:10:19,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:19,296 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:10:21,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:22,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:23,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:10:23,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:23,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:10:23,319 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:10:25,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:27,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:28,417 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:29,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:31,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:33,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:33,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:35,313 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:37,315 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:38,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:10:38,363 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:10:38,364 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:10:39,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:39,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:41,319 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:42,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:44,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:44,697 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:46,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:48,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:49,028 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:10:50,139 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:50,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:52,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:53,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:10:53,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:10:53,344 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:10:54,332 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:55,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:10:56,334 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:10:58,336 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:00,338 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:01,032 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:02,340 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:03,684 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:11:03,686 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:11:03,686 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:11:03,688 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:11:04,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:04,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:11:06,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:06,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:08,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:11:08,344 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:11:08,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:11:08,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:10,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:11,906 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:12,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:14,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:16,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:17,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:18,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:19,041 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:11:20,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:22,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:22,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:23,105 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:11:23,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:11:23,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:11:24,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:26,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:28,204 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:28,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:30,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:32,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:33,621 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:34,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:36,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:11:38,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:11:38,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:11:38,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:39,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:40,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:42,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:44,388 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:44,438 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:46,390 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:48,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:48,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:11:48,900 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:11:48,900 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:11:48,902 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:11:49,033 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:11:49,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:11:50,034 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:50,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:52,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:53,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:11:53,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:11:53,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:11:54,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:55,309 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:11:56,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:11:58,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:00,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:00,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:02,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:04,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:06,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:06,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:08,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:12:08,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:12:08,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:12:08,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:10,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:11,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:12,419 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:14,421 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:16,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:17,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:18,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:19,035 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:12:20,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:22,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:22,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:23,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:12:23,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:12:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:12:24,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:26,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:27,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:28,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:30,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:32,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:33,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:34,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:12:34,109 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:12:34,109 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:12:34,111 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:12:34,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:34,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:12:36,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:38,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:12:38,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:12:38,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:12:38,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:38,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:40,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:42,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:44,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:44,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:46,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:48,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:49,037 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:12:50,038 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:50,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:52,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:53,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:12:53,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:12:53,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:12:54,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:55,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:12:56,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:12:58,469 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:00,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:01,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:02,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:04,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:06,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:06,556 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:08,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:13:08,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:13:08,345 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:13:08,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:10,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:11,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:12,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:14,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:16,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:17,400 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:18,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:19,052 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:13:19,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:13:19,245 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:13:19,246 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:13:19,247 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:13:19,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:13:20,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:22,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:22,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:23,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:13:23,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:13:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:13:24,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:26,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:28,251 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:28,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:30,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:32,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:33,654 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:34,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:36,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:38,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:13:38,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:13:38,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:13:38,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:39,053 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:40,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:42,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:44,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:44,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:46,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:48,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:49,042 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:13:49,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:50,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:52,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:53,112 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:13:53,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:13:53,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:13:54,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:55,232 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:13:56,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:13:58,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:00,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:00,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:02,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:04,268 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:14:04,270 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:14:04,270 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:14:04,272 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:14:04,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:04,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:14:06,046 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:06,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:14:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:14:08,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:14:08,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:10,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:11,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:12,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:14,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:16,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:16,920 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:18,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:19,044 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:14:20,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:22,333 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:22,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:23,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:14:23,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:14:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:14:24,563 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:26,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:27,730 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:28,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:30,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:32,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:33,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:34,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:36,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:38,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:14:38,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:14:38,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:14:38,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:38,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:40,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:42,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:43,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:44,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:46,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:48,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:49,046 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:14:49,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:49,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:14:49,347 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:14:49,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:14:49,349 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:14:49,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:14:50,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:52,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:53,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:14:53,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:14:53,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:14:54,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:14:54,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:56,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:14:58,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:00,125 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:00,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:02,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:04,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:05,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:06,610 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:15:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:15:08,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:15:08,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:10,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:10,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:12,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:14,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:16,360 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:16,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:18,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:19,059 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:15:20,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:21,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:22,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:23,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:15:23,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:15:23,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:15:24,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:26,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:27,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:28,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:30,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:32,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:32,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:34,381 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:15:34,383 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:15:34,383 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:15:34,385 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:15:34,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:34,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:15:36,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:37,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:38,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:15:38,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:15:38,346 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:15:38,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:40,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:42,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:43,382 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:44,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:46,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:48,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:48,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:49,051 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:15:50,903 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:52,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:53,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:15:53,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:15:53,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:15:54,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:15:54,907 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:56,911 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:58,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:15:59,614 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:00,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:02,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:04,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:05,019 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:06,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:08,113 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:16:08,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:16:08,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:16:08,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:10,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:10,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:12,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:14,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:15,800 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:16,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:18,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:19,054 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:16:19,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:16:19,450 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:16:19,450 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:16:19,452 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:16:19,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:16:20,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:21,224 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:22,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:16:23,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:16:23,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:16:24,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:26,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:26,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:28,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:30,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:32,060 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:32,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:34,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:36,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:37,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:38,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:16:38,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:16:38,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:16:38,955 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:40,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:42,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:42,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:44,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:46,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:48,248 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:48,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:49,056 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:16:50,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:52,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:53,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:16:53,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:16:53,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:16:53,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:16:54,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:56,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:58,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:16:59,010 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:00,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:02,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:04,394 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:04,426 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:17:04,427 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:17:04,428 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:17:04,430 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:17:04,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:04,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:17:06,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:08,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:17:08,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:17:08,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:17:08,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:09,447 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:10,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:12,995 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:14,997 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:15,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:16,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:19,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:19,058 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:17:20,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:21,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:23,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:23,122 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:17:23,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:17:23,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:17:25,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:26,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:27,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:29,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:31,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:31,518 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:33,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:35,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:36,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:37,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:38,121 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:17:38,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:17:38,347 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:17:39,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:41,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:42,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:43,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:45,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:47,031 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:47,666 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:49,033 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:49,060 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:17:49,491 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:17:49,492 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:17:49,493 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:17:49,494 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:17:50,035 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:17:51,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:53,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:53,050 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:17:53,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:17:53,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:17:55,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:57,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:17:58,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:17:59,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:01,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:03,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:03,816 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:05,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:07,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:08,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:18:08,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:18:08,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:18:09,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:09,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:11,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:13,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:14,631 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:15,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:17,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:19,063 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:18:19,065 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:20,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:21,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:23,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:23,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:18:23,347 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:18:23,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:18:25,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:25,429 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:27,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:29,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:30,808 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:31,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:33,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:34,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:18:34,426 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:18:34,426 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:18:34,428 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:18:35,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:35,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:18:36,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:37,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:38,127 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:18:38,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:18:38,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:18:39,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:41,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:41,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:43,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:45,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:46,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:47,095 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:49,065 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:18:49,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:51,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:52,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:53,101 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:18:53,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:18:53,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:18:55,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:57,105 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:18:57,717 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:18:59,107 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:01,109 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:03,111 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:03,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:05,113 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:07,115 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:08,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:19:08,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:08,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:19:08,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:19:09,117 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:11,119 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:13,122 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:13,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:15,124 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:17,126 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:19,067 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:19:19,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:19,128 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:19,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:19:19,351 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:19:19,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:19:19,353 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:19:20,130 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:19:21,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:19:23,133 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:23,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:19:23,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:19:24,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:25,135 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:27,137 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:29,139 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:30,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:31,141 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:33,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:35,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:35,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:37,148 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:38,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:19:38,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:19:38,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:19:39,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:40,902 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:41,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:43,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:45,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:46,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:47,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:49,070 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:19:49,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:51,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:51,683 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:19:53,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:53,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:19:53,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:19:55,166 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:57,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:19:57,168 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:19:59,170 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:01,172 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:02,451 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:03,174 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:04,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:20:04,290 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:20:04,291 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:20:04,293 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:20:05,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:05,177 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:20:07,179 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:07,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:08,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:20:08,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:20:08,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:20:09,181 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:11,183 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:13,185 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:13,275 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:15,187 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:17,189 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:18,659 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:19,072 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:20:19,191 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:21,193 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:23,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:20:23,195 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:23,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:20:23,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:20:24,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:25,206 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:27,208 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:29,210 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:29,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:31,213 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:33,215 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:34,788 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:35,217 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:37,219 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:38,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:20:38,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:20:38,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:20:39,221 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:40,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:41,223 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:43,225 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:45,227 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:45,545 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:47,229 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:49,074 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:20:49,177 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:20:49,178 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:20:49,179 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:20:49,181 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:20:49,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:49,232 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:20:50,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:51,234 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:53,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:20:53,236 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:53,348 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:20:53,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:20:55,238 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:56,330 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:20:57,240 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:20:59,242 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:01,244 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:01,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:03,246 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:05,248 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:07,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:07,250 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:08,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:21:08,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:21:08,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:21:09,252 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:11,254 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:12,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:13,257 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:15,259 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:17,261 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:17,833 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:19,076 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:21:19,263 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:21,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:23,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:21:23,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:23,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:23,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:21:23,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:21:25,269 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:27,271 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:28,578 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:29,273 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:31,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:33,277 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:33,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:34,003 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:21:34,005 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:21:34,005 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:21:34,007 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:21:34,278 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:21:35,279 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:37,282 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:38,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:21:38,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:21:38,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:21:39,284 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:39,371 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:41,286 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:43,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:44,807 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:45,290 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:47,292 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:49,079 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:21:49,294 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:50,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:51,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:21:53,298 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:53,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:21:53,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:21:55,300 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:55,592 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:21:57,302 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:21:59,303 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:00,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:01,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:03,307 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:05,309 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:06,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:07,311 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:08,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:22:08,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:22:08,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:22:09,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:11,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:11,750 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:13,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:15,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:17,159 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:17,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:18,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:22:18,992 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:22:18,992 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:22:18,994 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:22:19,081 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:22:19,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:19,325 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:22:21,327 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:22,559 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:23,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:22:23,329 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:23,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:22:23,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:22:25,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:27,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:27,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:29,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:31,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:33,312 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:33,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:35,341 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:37,343 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:38,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:22:38,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:22:38,349 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:22:38,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:39,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:41,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:43,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:44,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:45,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:47,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:49,083 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:22:49,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:49,355 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:51,357 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:22:53,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:22:53,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:22:53,359 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:54,474 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:22:55,361 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:57,363 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:22:59,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:00,244 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:01,366 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:03,368 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:03,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:23:03,914 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:23:03,914 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:23:03,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:23:04,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:23:05,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:05,680 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:07,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:08,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:23:08,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:23:08,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:23:09,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:11,071 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:11,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:13,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:15,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:16,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:17,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:19,085 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:23:19,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:21,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:21,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:23,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:23:23,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:23:23,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:23:23,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:25,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:27,205 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:27,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:29,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:31,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:32,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:33,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:35,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:37,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:37,960 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:38,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:23:38,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:23:38,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:23:39,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:41,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:43,357 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:43,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:45,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:47,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:48,742 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:48,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:23:48,785 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:23:48,785 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:23:48,787 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:23:49,088 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:23:49,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:49,417 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:23:51,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:53,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:23:53,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:23:53,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:23:53,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:54,109 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:23:55,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:57,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:59,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:23:59,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:01,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:03,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:04,851 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:05,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:07,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:08,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:24:08,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:24:08,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:24:09,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:10,216 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:11,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:13,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:15,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:15,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:17,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:19,157 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:24:19,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:20,972 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:21,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:24:23,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:24:23,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:24:23,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:25,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:26,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:27,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:29,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:31,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:31,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:33,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:33,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:24:33,566 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:24:33,566 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:24:33,568 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:24:34,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:24:35,466 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:37,118 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:37,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:38,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:24:38,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:24:38,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:24:39,471 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:41,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:42,495 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:43,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:45,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:47,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:47,884 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:49,160 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:24:49,483 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:51,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:53,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:24:53,136 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:53,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:24:53,350 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:24:53,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:55,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:57,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:24:58,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:24:59,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:01,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:03,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:03,971 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:05,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:07,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:08,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:25:08,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:25:08,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:25:09,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:09,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:11,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:13,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:14,771 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:15,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:17,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:18,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:25:18,394 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:25:18,394 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:25:18,396 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:25:18,517 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:25:19,162 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:25:19,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:20,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:21,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:25:23,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:25:23,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:25:23,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:25,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:25,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:27,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:29,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:31,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:31,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:33,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:35,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:36,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:37,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:38,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:25:38,350 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:25:38,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:25:39,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:41,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:41,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:43,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:45,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:47,172 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:47,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:49,174 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:25:49,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:51,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:52,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:53,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:25:53,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:25:53,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:25:53,565 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:55,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:57,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:25:57,954 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:25:59,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:01,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:03,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:03,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:26:03,387 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:26:03,387 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:26:03,389 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:26:03,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:03,581 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:26:05,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:07,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:08,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:26:08,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:26:08,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:26:08,488 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:09,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:11,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:12,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:14,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:14,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:16,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:18,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:19,166 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:26:19,615 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:20,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:22,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:23,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:26:23,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:26:23,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:26:24,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:25,007 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:26,608 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:28,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:30,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:30,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:32,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:34,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:35,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:36,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:26:38,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:26:38,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:26:38,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:40,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:41,134 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:42,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:44,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:46,505 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:46,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:48,322 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:26:48,324 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:26:48,325 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:26:48,326 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:26:48,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:48,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:26:49,168 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:26:50,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:51,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:52,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:53,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:26:53,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:26:53,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:26:54,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:56,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:26:57,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:26:58,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:00,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:02,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:02,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:04,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:06,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:08,018 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:08,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:27:08,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:27:08,351 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:27:08,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:10,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:12,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:13,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:14,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:16,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:18,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:18,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:19,171 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:27:20,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:22,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:23,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:27:23,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:27:23,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:27:23,811 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:24,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:26,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:28,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:29,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:30,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:32,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:33,156 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:27:33,159 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:27:33,159 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:27:33,160 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:27:33,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:27:34,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:34,923 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:36,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:27:38,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:27:38,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:27:38,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:40,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:40,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:42,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:44,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:45,638 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:46,693 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:48,695 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:49,183 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:27:50,697 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:50,990 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:52,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:53,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:27:53,351 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:27:53,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:27:54,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:56,362 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:27:56,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:27:58,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:00,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:01,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:02,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:04,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:06,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:07,130 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:08,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:28:08,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:28:08,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:28:08,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:10,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:12,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:12,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:14,722 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:16,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:17,882 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:17,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:28:17,915 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:28:17,916 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:28:17,917 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:28:18,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:18,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:28:19,175 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:28:20,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:22,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:23,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:28:23,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:23,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:28:23,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:28:24,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:26,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:28,623 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:28,737 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:30,739 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:32,741 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:33,989 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:34,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:36,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:38,131 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:28:38,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:28:38,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:28:38,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:39,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:40,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:42,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:44,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:44,761 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:46,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:48,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:49,177 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:28:50,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:50,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:52,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:53,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:28:53,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:28:53,352 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:28:54,763 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:55,506 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:28:56,765 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:28:58,767 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:00,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:00,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:02,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:29:02,692 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:29:02,693 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:29:02,695 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:29:02,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:02,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:29:04,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:06,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:06,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:08,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:29:08,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:29:08,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:29:08,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:10,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:11,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:12,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:14,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:16,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:16,962 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:18,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:19,179 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:29:20,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:22,319 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:22,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:23,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:29:23,352 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:29:23,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:29:24,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:26,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:27,810 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:28,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:30,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:32,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:33,176 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:34,805 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:36,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:38,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:29:38,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:29:38,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:29:38,500 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:38,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:40,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:42,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:43,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:44,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:46,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:47,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:29:47,603 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:29:47,604 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:29:47,605 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:29:47,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:29:48,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:49,192 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:29:49,193 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:50,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:52,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:53,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:29:53,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:29:53,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:29:54,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:29:54,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:56,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:29:58,830 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:00,140 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:00,832 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:02,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:04,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:05,508 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:06,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:08,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:30:08,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:30:08,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:30:08,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:10,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:10,869 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:12,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:14,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:16,243 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:16,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:18,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:19,184 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:30:20,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:21,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:22,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:23,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:30:23,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:30:23,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:30:24,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:26,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:27,009 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:28,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:30,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:32,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:32,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:30:32,457 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:30:32,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:30:32,460 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:30:32,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:32,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:30:34,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:36,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:37,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:38,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:30:38,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:30:38,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:30:38,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:40,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:42,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:43,220 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:44,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:46,880 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:48,591 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:48,882 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:49,186 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:30:50,884 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:52,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:53,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:30:53,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:30:53,353 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:30:53,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:30:54,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:56,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:58,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:30:59,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:00,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:02,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:04,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:04,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:06,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:08,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:31:08,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:31:08,386 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:31:08,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:10,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:10,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:12,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:14,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:15,483 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:16,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:17,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:31:17,312 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:31:17,312 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:31:17,314 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:31:17,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:31:18,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:19,188 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:31:20,880 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:20,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:22,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:23,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:31:23,372 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:31:23,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:31:24,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:26,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:26,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:28,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:30,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:31,611 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:32,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:34,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:36,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:36,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:38,133 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:31:38,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:31:38,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:31:38,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:40,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:42,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:42,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:44,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:46,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:47,690 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:48,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:49,191 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:31:50,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:52,947 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:53,066 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:53,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:31:53,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:31:53,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:31:54,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:56,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:31:58,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:31:58,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:00,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:02,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:32:02,084 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:32:02,085 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:32:02,086 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:32:02,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:02,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:32:03,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:04,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:06,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:08,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:32:08,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:32:08,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:32:08,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:09,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:10,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:12,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:14,627 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:14,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:16,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:18,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:19,193 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:32:20,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:20,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:22,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:23,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:32:23,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:32:23,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:32:24,981 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:25,439 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:26,983 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:28,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:30,823 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:30,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:32,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:34,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:36,200 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:36,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:38,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:32:38,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:32:38,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:32:38,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:40,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:41,576 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:43,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:45,002 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:46,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:46,992 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:32:46,994 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:32:46,994 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:32:46,996 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:32:47,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:47,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:32:49,006 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:49,195 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:32:51,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:52,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:53,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:53,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:32:53,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:32:53,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:32:55,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:57,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:32:57,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:32:59,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:01,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:03,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:03,189 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:05,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:07,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:08,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:33:08,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:33:08,373 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:33:08,536 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:09,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:11,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:13,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:13,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:15,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:17,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:19,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:19,198 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:33:19,199 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:21,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:23,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:23,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:33:23,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:33:23,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:33:24,516 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:25,041 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:27,043 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:29,045 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:30,080 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:31,047 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:31,908 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:33:31,909 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:33:31,910 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:33:31,911 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:33:32,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:33:33,050 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:35,052 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:35,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:37,054 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:38,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:33:38,373 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:33:38,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:33:39,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:40,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:41,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:43,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:45,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:46,247 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:47,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:49,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:49,200 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:33:51,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:51,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:53,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:53,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:33:53,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:33:53,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:33:55,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:56,969 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:33:57,074 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:33:59,076 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:01,078 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:02,332 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:03,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:05,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:07,083 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:07,701 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:08,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:34:08,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:34:08,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:34:09,086 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:11,088 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:13,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:13,090 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:15,092 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:16,685 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:34:16,687 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:34:16,687 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:34:16,689 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:34:17,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:17,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:34:18,462 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:19,099 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:19,202 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:34:21,103 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:23,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:23,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:34:23,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:34:23,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:34:23,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:25,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:27,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:29,110 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:29,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:31,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:33,114 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:34,577 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:35,116 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:37,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:38,144 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:34:38,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:34:38,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:34:39,120 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:39,939 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:41,123 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:43,125 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:45,127 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:45,315 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:47,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:49,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:49,204 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:34:50,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:51,134 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:53,136 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:53,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:34:53,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:34:53,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:34:55,138 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:56,082 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:34:57,140 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:34:59,142 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:01,144 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:01,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:01,475 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:35:01,477 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:35:01,478 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:35:01,479 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:35:02,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:35:03,146 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:05,150 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:06,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:07,152 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:35:08,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:35:08,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:35:09,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:11,155 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:12,185 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:13,157 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:15,159 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:17,161 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:17,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:19,164 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:19,207 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:35:21,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:22,897 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:23,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:35:23,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:23,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:35:23,374 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:35:25,169 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:27,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:28,282 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:29,429 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:31,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:33,433 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:33,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:35,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:37,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:38,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:35:38,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:35:38,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:35:39,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:39,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:41,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:43,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:44,418 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:45,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:46,241 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:35:46,243 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:35:46,243 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:35:46,245 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:35:46,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:35:47,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:49,209 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:35:49,453 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:49,794 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:51,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:53,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:35:53,409 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:35:53,409 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:35:53,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:55,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:35:55,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:57,462 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:35:59,464 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:00,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:01,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:03,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:05,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:05,919 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:07,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:36:08,374 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:36:08,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:36:09,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:11,353 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:11,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:13,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:15,485 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:16,738 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:17,487 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:19,213 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:36:19,489 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:21,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:22,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:23,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:36:23,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:36:23,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:36:23,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:25,496 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:27,478 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:27,498 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:29,500 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:31,090 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:36:31,092 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:36:31,093 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:36:31,094 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:36:31,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:31,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:36:32,854 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:33,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:35,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:37,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:38,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:36:38,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:38,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:36:38,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:36:39,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:41,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:43,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:43,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:45,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:47,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:48,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:49,214 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:36:49,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:51,525 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:53,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:36:53,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:36:53,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:36:53,527 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:54,393 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:36:55,529 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:57,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:59,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:36:59,773 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:01,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:03,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:05,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:05,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:07,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:08,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:37:08,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:37:08,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:37:09,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:10,507 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:11,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:13,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:15,550 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:15,873 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:15,904 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:37:15,906 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:37:15,906 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:37:15,908 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:37:16,552 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:37:17,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:19,217 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:37:19,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:21,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:21,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:37:23,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:37:23,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:37:23,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:25,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:26,596 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:27,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:29,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:31,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:31,953 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:33,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:35,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:37,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:37,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:38,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:37:38,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:37:38,375 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:37:39,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:41,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:42,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:43,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:45,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:47,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:48,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:49,219 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:37:49,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:51,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:53,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:37:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:53,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:37:53,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:37:53,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:55,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:57,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:37:58,825 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:37:59,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:00,645 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:38:00,647 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:38:00,647 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:38:00,649 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:38:01,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:01,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:38:03,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:04,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:05,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:07,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:08,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:38:08,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:38:08,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:38:09,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:09,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:11,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:13,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:14,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:15,620 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:17,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:19,234 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:38:19,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:20,236 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:21,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:38:23,375 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:38:23,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:38:23,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:25,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:25,698 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:27,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:29,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:31,089 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:31,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:33,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:35,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:36,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:37,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:38,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:38:38,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:38:38,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:38:39,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:41,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:41,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:43,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:45,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:38:45,455 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:38:45,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:38:45,457 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:38:45,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:45,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:38:47,213 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:47,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:49,223 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:38:49,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:51,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:52,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:53,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:38:53,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:38:53,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:38:53,667 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:55,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:57,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:38:57,929 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:38:59,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:01,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:03,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:03,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:05,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:07,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:08,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:39:08,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:39:08,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:39:08,525 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:09,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:11,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:13,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:13,994 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:15,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:17,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:19,226 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:39:19,227 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:19,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:21,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:23,146 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:39:23,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:39:23,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:39:23,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:24,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:25,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:27,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:29,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:30,135 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:30,167 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:39:30,169 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:39:30,169 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:39:30,170 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:39:30,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:39:31,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:33,713 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:35,564 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:35,715 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:37,717 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:38,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:39:38,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:39:38,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:39:39,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:40,979 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:41,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:43,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:45,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:46,349 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:47,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:49,228 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:39:49,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:51,719 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:51,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:39:53,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:39:53,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:39:53,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:55,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:57,084 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:39:57,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:39:59,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:01,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:02,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:03,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:05,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:07,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:07,801 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:08,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:40:08,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:40:08,376 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:40:09,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:11,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:13,162 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:13,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:14,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:40:14,990 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:40:14,990 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:40:14,992 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:40:15,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:15,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:40:17,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:18,542 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:19,242 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:40:19,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:21,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:23,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:40:23,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:40:23,377 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:40:23,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:23,931 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:25,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:27,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:29,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:29,772 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:31,774 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:33,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:34,734 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:35,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:37,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:38,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:40:38,406 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:40:38,407 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:40:39,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:40,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:41,785 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:43,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:45,477 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:45,789 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:47,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:49,233 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:40:49,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:50,853 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:51,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:40:53,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:40:53,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:40:53,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:55,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:56,225 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:40:57,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:59,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:40:59,842 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:40:59,844 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:40:59,844 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:40:59,847 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:41:00,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:41:01,604 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:01,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:03,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:05,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:06,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:07,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:08,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:41:08,387 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:41:08,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:41:09,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:11,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:12,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:13,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:15,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:17,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:17,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:19,235 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:41:19,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:21,828 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:23,123 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:23,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:41:23,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:41:23,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:41:23,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:25,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:27,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:28,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:29,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:31,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:33,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:33,916 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:35,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:37,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:41:38,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:41:38,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:41:39,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:41,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:43,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:44,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:44,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:41:44,691 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:41:44,691 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:41:44,693 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:41:44,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:41:45,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:47,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:49,237 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:41:49,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:50,020 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:51,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:53,147 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:41:53,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:41:53,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:41:53,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:55,384 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:41:55,865 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:57,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:41:58,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:00,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:00,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:02,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:04,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:06,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:06,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:42:08,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:42:08,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:42:08,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:10,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:11,461 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:12,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:14,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:16,832 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:16,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:18,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:19,249 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:42:20,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:22,219 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:22,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:42:23,388 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:42:23,388 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:42:24,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:26,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:27,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:28,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:29,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:42:29,425 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:42:29,425 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:42:29,427 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:42:29,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:42:30,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:31,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:32,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:32,984 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:34,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:36,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:42:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:38,443 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:42:38,443 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:42:38,914 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:40,916 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:42,918 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:43,722 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:44,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:46,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:48,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:49,075 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:49,241 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:42:50,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:52,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:42:53,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:42:53,423 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:42:54,446 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:42:54,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:56,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:58,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:42:59,822 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:00,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:02,942 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:04,944 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:05,214 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:06,946 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:08,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:43:08,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:43:08,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:43:08,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:10,599 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:10,951 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:12,953 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:14,218 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:43:14,220 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:43:14,221 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:43:14,222 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:43:14,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:14,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:43:15,957 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:15,980 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:16,959 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:18,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:19,244 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:43:20,964 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:21,354 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:22,966 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:43:23,423 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:43:23,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:43:24,968 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:26,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:26,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:28,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:30,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:32,067 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:32,976 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:34,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:36,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:37,435 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:38,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:43:38,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:43:38,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:43:38,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:40,985 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:42,809 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:42,987 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:44,989 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:46,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:48,190 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:48,993 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:49,246 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:43:50,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:52,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:53,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:43:53,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:43:53,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:43:53,535 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:55,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:57,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:59,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:43:59,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:43:59,031 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:43:59,033 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:43:59,033 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:43:59,035 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:44:00,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:44:01,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:02,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:03,010 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:04,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:05,012 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:07,014 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:44:08,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:44:08,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:44:09,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:09,566 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:11,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:13,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:15,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:15,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:17,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:19,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:19,258 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:44:20,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:21,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:23,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:23,148 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:44:23,424 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:44:23,424 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:44:25,036 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:25,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:27,038 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:29,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:31,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:31,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:33,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:35,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:36,588 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:37,049 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:44:38,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:44:38,473 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:44:39,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:41,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:41,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:43,056 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:43,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:44:43,828 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:44:43,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:44:43,830 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:44:44,058 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:44:45,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:46,060 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:47,061 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:47,401 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:49,063 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:49,250 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:44:51,066 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:52,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:53,069 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:44:53,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:44:53,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:44:55,071 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:57,073 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:44:58,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:44:59,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:01,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:03,079 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:03,574 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:05,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:07,084 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:45:08,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:45:08,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:45:08,603 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:09,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:11,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:13,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:14,337 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:15,093 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:17,096 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:19,098 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:19,252 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:45:19,700 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:21,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:23,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:45:23,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:45:23,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:45:25,061 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:25,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:27,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:28,667 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:45:28,669 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:45:28,669 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:45:28,671 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:45:29,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:29,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:45:30,433 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:31,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:33,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:35,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:35,814 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:37,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:38,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:45:38,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:45:38,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:45:39,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:41,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:41,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:43,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:45,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:46,620 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:47,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:49,255 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:45:49,401 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:51,403 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:52,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:53,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:45:53,405 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:53,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:45:53,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:45:55,407 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:57,392 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:45:57,409 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:45:59,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:01,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:02,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:03,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:05,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:07,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:08,137 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:08,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:46:08,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:46:08,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:46:09,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:11,427 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:13,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:13,520 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:13,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:46:13,551 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:46:13,551 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:46:13,552 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:46:14,431 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:46:15,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:17,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:18,942 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:19,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:46:19,437 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:21,439 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:23,149 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:46:23,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:23,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:46:23,453 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:46:24,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:25,443 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:27,445 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:29,447 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:29,765 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:31,449 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:33,451 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:35,145 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:35,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:37,456 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:46:38,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:46:38,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:46:39,458 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:40,521 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:41,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:43,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:45,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:45,887 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:47,468 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:49,259 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:46:49,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:51,254 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:51,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:46:53,453 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:46:53,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:46:53,475 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:55,477 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:56,629 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:46:57,479 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:46:58,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:46:58,459 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:46:58,459 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:46:58,461 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:46:58,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:46:59,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:01,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:02,023 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:03,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:05,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:07,419 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:07,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:47:08,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:47:08,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:47:09,492 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:11,494 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:12,805 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:13,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:15,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:17,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:18,182 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:19,261 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:47:19,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:21,507 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:47:23,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:47:23,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:47:23,509 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:23,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:25,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:27,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:28,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:29,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:31,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:33,521 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:34,266 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:35,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:37,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:47:38,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:47:38,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:47:39,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:39,548 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:41,531 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:43,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:47:43,211 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:47:43,211 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:47:43,213 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:47:43,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:43,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:47:44,951 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:45,536 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:47,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:49,263 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:47:49,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:50,264 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:51,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:47:53,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:47:53,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:47:53,546 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:55,548 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:55,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:47:57,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:47:59,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:00,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:01,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:03,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:05,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:06,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:07,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:08,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:48:08,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:48:08,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:48:09,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:11,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:11,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:13,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:15,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:16,967 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:17,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:19,266 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:48:19,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:21,577 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:22,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:23,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:48:23,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:48:23,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:48:23,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:25,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:27,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:27,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:27,640 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:48:27,642 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:48:27,642 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:48:27,644 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:48:28,586 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:48:29,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:31,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:32,936 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:33,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:35,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:37,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:38,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:48:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:38,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:48:38,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:48:39,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:41,599 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:43,601 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:43,606 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:45,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:47,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:48,952 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:49,268 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:48:49,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:51,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:53,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:48:53,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:48:53,454 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:48:53,612 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:54,411 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:48:55,614 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:57,616 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:59,618 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:48:59,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:01,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:03,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:05,055 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:05,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:07,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:08,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:49:08,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:49:08,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:49:09,630 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:10,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:11,632 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:12,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:49:12,166 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:49:12,166 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:49:12,168 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:49:12,634 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:49:13,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:15,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:15,686 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:17,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:19,270 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:49:19,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:20,999 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:21,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:23,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:49:23,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:49:23,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:49:23,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:25,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:26,306 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:27,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:29,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:31,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:31,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:33,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:35,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:36,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:37,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:49:38,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:49:38,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:49:39,663 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:41,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:42,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:43,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:45,669 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:47,636 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:47,672 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:49,274 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:49:49,674 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:51,676 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:52,964 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:53,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:49:53,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:49:53,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:49:53,678 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:55,680 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:56,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:49:56,541 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:49:56,542 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:49:56,544 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:49:56,682 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:49:57,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:49:58,288 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:49:59,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:01,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:03,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:03,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:05,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:07,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:08,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:50:08,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:50:08,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:50:08,622 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:09,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:11,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:13,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:14,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:15,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:17,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:19,276 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:50:19,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:19,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:21,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:23,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:50:23,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:50:23,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:50:23,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:24,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:25,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:27,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:29,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:30,271 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:31,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:33,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:35,612 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:35,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:37,724 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:50:38,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:50:38,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:50:39,726 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:40,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:40,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:50:40,979 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:50:40,980 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:50:40,981 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:50:41,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:41,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:50:43,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:45,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:46,273 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:47,735 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:49,279 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:50:49,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:51,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:51,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:53,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:50:53,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:50:53,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:50:53,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:55,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:56,903 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:50:57,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:50:59,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:01,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:02,201 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:03,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:05,754 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:07,515 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:07,756 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:08,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:51:08,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:51:08,455 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:51:09,758 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:11,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:12,835 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:13,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:15,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:17,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:18,165 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:19,281 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:51:19,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:21,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:23,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:51:23,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:51:23,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:51:23,601 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:23,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:25,316 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:51:25,317 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:51:25,318 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:51:25,321 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:51:25,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:25,776 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:51:27,778 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:28,841 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:29,780 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:31,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:33,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:34,163 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:35,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:37,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:38,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:51:38,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:51:38,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:51:39,479 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:39,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:41,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:43,794 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:44,796 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:45,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:47,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:49,283 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:51:49,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:50,107 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:51,802 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:51:53,455 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:51:53,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:51:53,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:55,436 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:51:55,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:57,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:51:59,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:00,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:01,812 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:03,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:05,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:06,114 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:07,818 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:08,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:52:08,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:52:08,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:52:09,695 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:52:09,697 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:52:09,697 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:52:09,699 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:52:09,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:09,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:52:11,449 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:11,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:13,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:15,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:16,783 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:17,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:19,285 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:52:19,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:21,833 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:22,100 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:23,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:52:23,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:52:23,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:52:23,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:25,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:27,404 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:27,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:29,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:31,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:32,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:33,844 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:35,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:37,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:38,024 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:38,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:52:38,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:52:38,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:52:39,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:41,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:43,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:43,854 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:45,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:47,858 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:48,679 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:49,298 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:52:49,860 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:51,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:52:53,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:52:53,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:52:53,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:54,026 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:54,058 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:52:54,059 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:52:54,060 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:52:54,062 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:52:54,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:52:55,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:57,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:52:59,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:52:59,872 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:01,874 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:03,876 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:04,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:05,878 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:07,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:08,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:53:08,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:53:08,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:53:09,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:10,039 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:11,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:13,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:15,358 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:15,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:17,890 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:19,290 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:53:19,892 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:20,677 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:21,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:23,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:53:23,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:53:23,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:53:23,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:25,898 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:26,014 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:27,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:29,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:31,343 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:31,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:33,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:35,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:36,676 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:37,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:38,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:53:38,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:53:38,494 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:53:38,495 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:53:38,586 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:53:38,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:53:38,588 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:53:38,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:53:39,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:41,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:42,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:43,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:45,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:47,339 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:47,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:49,292 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:53:49,922 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:51,924 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:52,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:53,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:53:53,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:53:53,456 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:53:53,927 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:55,929 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:57,931 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:53:57,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:53:59,933 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:01,935 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:03,292 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:03,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:05,939 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:07,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:08,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:54:08,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:54:08,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:54:08,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:09,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:11,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:13,937 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:13,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:15,949 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:17,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:19,276 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:19,294 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:54:19,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:21,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:22,875 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:54:22,877 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:54:22,877 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:54:22,879 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:54:22,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:54:23,152 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:54:23,456 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:54:23,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:54:23,961 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:24,600 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:25,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:27,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:29,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:29,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:31,969 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:33,971 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:35,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:35,973 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:37,975 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:38,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:54:38,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:54:38,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:54:39,977 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:40,608 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:41,979 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:43,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:45,934 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:45,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:47,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:49,308 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:54:49,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:51,245 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:51,991 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:53,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:54:53,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:54:53,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:54:53,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:55,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:54:56,561 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:54:57,998 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:00,000 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:01,878 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:02,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:04,004 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:06,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:07,209 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:07,240 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:55:07,242 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:55:07,242 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:55:07,244 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:55:08,008 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:08,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:55:08,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:55:08,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:55:08,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:55:10,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:12,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:12,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:14,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:16,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:17,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:18,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:19,299 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:55:20,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:22,025 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:23,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:55:23,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:23,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:55:23,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:55:24,027 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:26,029 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:28,032 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:28,558 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:29,288 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:31,291 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:33,293 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:33,872 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:35,295 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:37,297 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:38,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:55:38,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:55:38,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:55:39,179 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:39,299 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:41,301 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:43,304 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:44,482 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:45,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:47,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:49,301 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:55:49,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:49,797 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:51,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:51,598 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:55:51,600 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:55:51,601 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:55:51,603 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:55:52,317 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:55:53,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:55:53,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:53,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:55:53,457 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:55:55,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:55:55,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:57,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:55:59,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:00,450 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:01,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:03,331 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:05,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:05,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:07,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:08,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:56:08,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:56:08,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:56:09,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:11,087 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:11,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:13,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:15,345 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:16,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:17,347 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:19,304 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:56:19,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:21,351 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:21,815 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:23,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:56:23,353 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:23,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:56:23,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:56:25,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:27,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:27,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:29,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:31,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:32,496 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:33,364 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:35,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:36,098 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:56:36,100 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:56:36,101 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:56:36,103 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:56:36,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:56:37,370 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:37,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:38,157 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:56:38,457 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:56:38,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:56:39,372 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:41,374 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:43,196 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:43,376 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:45,378 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:47,380 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:48,510 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:49,316 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:56:49,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:51,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:53,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:56:53,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:53,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:56:53,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:56:53,529 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:55,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:57,391 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:56:59,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:56:59,393 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:01,395 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:03,398 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:04,460 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:05,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:07,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:08,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:57:08,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:57:08,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:57:09,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:09,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:11,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:13,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:15,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:15,411 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:17,413 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:19,308 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:57:19,415 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:20,310 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:20,459 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:57:20,461 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:57:20,461 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:57:20,463 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:57:21,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:21,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:57:23,154 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:57:23,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:23,458 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:57:23,458 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:57:25,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:26,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:27,425 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:29,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:31,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:31,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:33,435 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:36,745 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:39,775 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:57:39,776 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:57:39,776 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:57:42,467 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:43,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:45,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:47,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:48,405 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:49,311 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:57:49,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:51,454 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:53,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:54,281 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:57:54,753 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:57:54,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:57:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:57:55,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:57,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:57:59,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:00,119 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:01,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:03,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:05,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:05,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:07,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:09,474 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:58:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:58:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:58:11,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:11,743 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:13,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:15,481 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:16,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:58:16,529 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:58:16,529 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:58:16,531 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:58:17,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:17,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:58:17,531 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:19,313 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:58:19,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:21,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:23,252 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:23,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:24,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:58:24,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:58:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:58:25,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:27,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:29,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:29,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:31,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:33,502 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:34,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:35,504 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:37,506 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:39,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:39,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:58:39,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:58:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:58:39,876 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:41,511 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:43,513 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:45,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:45,515 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:47,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:49,327 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:58:49,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:50,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:51,523 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:53,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:58:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:58:54,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:58:55,533 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:55,911 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:58:57,535 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:58:59,538 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:01,509 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:01,540 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:03,542 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:04,329 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:59:04,331 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:59:04,331 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:59:04,333 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:59:04,544 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:59:05,545 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:07,198 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:07,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:09,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:59:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:59:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:59:11,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:12,910 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:13,554 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:15,556 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:17,558 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:18,616 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:19,317 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:59:19,560 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:21,562 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:23,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:24,303 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:24,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:59:24,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:59:24,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:59:25,567 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:27,569 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:29,571 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:29,988 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:31,573 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:33,575 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:35,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:35,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:37,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:39,582 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:39,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:59:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:59:39,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:59:41,304 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:41,584 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:43,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:45,589 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:46,959 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:47,591 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:49,320 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 15:59:49,593 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:51,595 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:51,630 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 15:59:51,631 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 15:59:51,632 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 15:59:51,633 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 15:59:52,597 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 15:59:52,634 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:53,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:54,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 15:59:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 15:59:54,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 15:59:55,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:57,602 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 15:59:58,258 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 15:59:59,604 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:01,606 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:03,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:03,905 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:05,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:07,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:09,587 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:09,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:09,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:00:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:00:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:00:11,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:13,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:15,278 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:15,622 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:17,624 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:19,322 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:00:19,626 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:20,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:21,628 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:23,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:00:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:00:24,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:00:25,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:26,533 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:27,635 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:29,637 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:31,639 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:32,164 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:33,641 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:35,643 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:37,645 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:37,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:38,652 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:00:38,654 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:00:38,654 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:00:38,655 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:00:39,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:39,649 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:00:39,754 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:00:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:00:39,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:00:41,651 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:43,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:43,653 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:45,655 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:47,657 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:48,961 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:49,334 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:00:49,659 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:51,661 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:53,664 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:54,549 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:00:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:00:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:00:54,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:00:55,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:57,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:00:59,670 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:00,128 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:01,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:03,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:05,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:05,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:07,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:09,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:01:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:01:09,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:01:11,277 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:11,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:13,685 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:15,687 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:16,861 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:17,689 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:19,327 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:01:19,691 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:21,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:22,440 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:23,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:01:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:01:24,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:01:25,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:01:25,207 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:01:25,207 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:01:25,209 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:01:25,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:25,699 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:01:27,701 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:28,044 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:29,703 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:31,705 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:33,648 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:33,707 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:35,709 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:37,711 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:39,259 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:39,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:01:39,755 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:01:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:01:41,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:43,718 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:44,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:45,720 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:47,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:49,329 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:01:49,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:50,427 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:51,727 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:53,729 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:01:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:01:54,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:01:55,732 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:56,029 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:01:57,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:01:59,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:01,609 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:01,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:03,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:05,743 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:07,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:07,745 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:09,747 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:09,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:02:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:02:09,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:02:11,749 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:11,787 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:02:11,788 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:02:11,789 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:02:11,792 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:02:12,751 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:02:12,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:13,753 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:15,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:17,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:18,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:19,330 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:02:19,759 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:21,761 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:23,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:23,898 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:24,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:02:24,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:02:24,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:02:25,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:27,768 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:29,468 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:29,770 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:31,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:33,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:35,035 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:35,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:37,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:39,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:02:39,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:02:39,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:02:39,782 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:40,901 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:41,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:43,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:45,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:46,124 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:47,790 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:49,332 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:02:49,792 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:51,644 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:51,795 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:53,797 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:54,755 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:02:54,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:02:54,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:02:55,799 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:57,188 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:02:57,801 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:02:58,065 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:02:58,067 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:02:58,067 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:02:58,069 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:02:58,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:02:59,804 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:01,807 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:02,721 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:03,809 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:05,811 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:07,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:08,297 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:03:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:03:09,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:03:09,815 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:11,817 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:13,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:13,870 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:15,821 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:17,823 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:19,335 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:03:19,335 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:19,825 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:21,827 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:23,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:24,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:03:24,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:03:24,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:03:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:25,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:27,835 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:29,837 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:30,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:31,839 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:33,841 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:35,843 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:36,033 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:37,845 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:39,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:03:39,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:03:39,756 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:03:39,847 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:41,562 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:41,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:43,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:44,289 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:03:44,290 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:03:44,290 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:03:44,292 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:03:44,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:03:45,853 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:47,103 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:47,856 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:49,337 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:03:49,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:51,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:52,610 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:53,862 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:54,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:03:54,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:03:54,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:03:55,864 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:57,866 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:03:58,120 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:03:59,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:01,870 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:03,647 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:03,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:05,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:07,885 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:09,155 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:09,756 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:04:09,757 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:04:09,757 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:04:09,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:11,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:13,891 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:14,646 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:15,893 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:17,896 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:19,339 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:04:19,900 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:20,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:21,902 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:23,904 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:24,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:04:24,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:04:24,799 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:04:25,906 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:25,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:27,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:29,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:30,246 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:04:30,248 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:04:30,248 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:04:30,250 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:04:30,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:04:31,250 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:31,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:33,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:35,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:36,802 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:37,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:39,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:04:39,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:04:39,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:04:39,921 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:41,923 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:42,364 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:43,925 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:45,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:47,874 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:47,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:49,341 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:04:49,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:51,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:53,386 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:53,937 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:54,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:04:54,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:04:54,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:04:55,940 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:57,941 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:04:58,885 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:04:59,943 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:01,945 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:03,948 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:04,412 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:05,950 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:07,952 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:09,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:05:09,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:05:09,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:05:09,930 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:09,954 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:11,956 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:13,958 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:15,448 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:15,960 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:16,324 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:05:16,326 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:05:16,327 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:05:16,328 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:05:16,962 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:05:17,963 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:19,343 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:05:19,965 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:20,983 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:21,967 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:23,970 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:24,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:05:24,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:05:24,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:05:25,972 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:26,593 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:27,974 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:29,978 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:31,980 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:32,207 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:33,982 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:35,984 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:37,737 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:37,986 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:39,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:05:39,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:05:39,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:05:39,988 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:41,990 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:43,234 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:43,992 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:45,994 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:47,996 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:48,718 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:49,346 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:05:49,999 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:52,001 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:54,003 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:54,206 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:05:54,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:05:54,778 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:05:54,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:05:56,005 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:58,007 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:05:59,671 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:00,009 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:02,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:02,346 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:06:02,348 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:06:02,348 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:06:02,350 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:06:03,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:06:04,016 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:05,132 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:06,018 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:08,020 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:09,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:06:09,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:06:09,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:06:10,022 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:10,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:12,024 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:14,026 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:16,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:16,030 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:18,030 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:19,348 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:06:20,034 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:21,487 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:22,037 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:24,040 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:24,778 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:06:24,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:06:24,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:06:26,042 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:27,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:28,044 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:30,046 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:32,048 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:32,485 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:34,051 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:36,053 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:37,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:38,055 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:39,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:06:39,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:06:39,791 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:06:40,057 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:42,059 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:43,497 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:44,062 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:46,064 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:48,047 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:06:48,049 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:06:48,050 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:06:48,051 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:06:48,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:48,068 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:06:49,052 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:49,351 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:06:50,070 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:52,072 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:54,075 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:54,532 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:06:54,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:06:54,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:06:54,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:06:56,077 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:06:58,080 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:00,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:00,082 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:02,085 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:04,087 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:05,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:06,089 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:08,091 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:09,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:07:09,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:07:09,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:07:10,097 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:10,963 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:12,100 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:14,102 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:16,104 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:16,454 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:18,106 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:19,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:07:20,108 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:21,924 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:22,165 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:24,167 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:24,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:07:24,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:07:24,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:07:26,182 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:27,420 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:28,192 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:30,265 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:32,267 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:32,914 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:33,772 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:07:33,774 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:07:33,774 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:07:33,775 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:07:34,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:34,275 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:07:36,281 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:38,305 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:38,385 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:39,306 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:39,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:07:39,779 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:07:39,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:07:41,308 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:43,310 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:43,843 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:45,312 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:47,314 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:49,294 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:49,316 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:49,355 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:07:51,318 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:53,320 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:54,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:07:54,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:07:54,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:07:54,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:07:55,322 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:57,324 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:07:59,326 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:00,208 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:01,328 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:03,330 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:05,333 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:05,681 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:07,335 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:09,337 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:09,779 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:08:09,780 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:08:09,792 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:08:11,150 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:11,339 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:13,342 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:15,344 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:16,605 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:17,346 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:19,263 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:08:19,265 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:08:19,265 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:08:19,268 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:08:19,348 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:19,349 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:08:19,357 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:08:21,350 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:22,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:23,352 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:24,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:08:24,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:08:24,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:08:25,354 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:27,356 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:27,492 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:29,358 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:31,360 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:32,955 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:33,362 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:35,365 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:37,367 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:38,415 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:39,369 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:39,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:08:39,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:08:39,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:08:41,371 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:43,373 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:43,883 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:45,375 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:47,377 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:49,345 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:49,359 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:08:49,379 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:51,381 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:53,383 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:54,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:08:54,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:08:54,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:08:54,863 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:08:55,385 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:57,387 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:08:59,389 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:00,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:01,392 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:03,394 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:04,724 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:09:04,726 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:09:04,726 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:09:04,728 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:09:05,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:05,397 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:09:05,728 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:07,399 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:09,400 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:09,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:09:09,784 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:09:09,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:09:11,153 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:11,402 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:13,404 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:15,406 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:16,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:17,408 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:19,372 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:09:19,410 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:21,412 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:22,083 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:23,414 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:24,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:09:24,849 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:09:24,849 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:09:25,416 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:27,418 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:27,550 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:29,420 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:31,422 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:33,017 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:33,424 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:35,426 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:37,428 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:38,473 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:39,430 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:39,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:09:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:09:39,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:09:41,432 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:43,434 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:43,917 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:45,436 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:47,438 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:49,364 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:09:49,365 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:49,440 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:50,217 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:09:50,219 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:09:50,220 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:09:50,221 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:09:50,441 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:09:51,442 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:53,444 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:54,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:09:54,828 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:09:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:09:54,829 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:09:55,446 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:57,448 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:09:59,450 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:00,290 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:01,452 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:03,455 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:05,457 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:05,768 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:07,459 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:09,461 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:10:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:10:09,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:10:11,226 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:11,463 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:13,465 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:15,467 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:16,689 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:17,470 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:19,366 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:10:19,472 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:21,473 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:22,151 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:23,476 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:10:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:10:24,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:10:25,478 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:27,480 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:27,633 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:29,482 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:31,484 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:33,079 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:33,486 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:35,488 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:35,752 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:10:35,753 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:10:35,754 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:10:35,755 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:10:36,490 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:10:37,491 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:38,534 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:39,493 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:10:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:10:39,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:10:41,495 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:43,497 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:43,993 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:45,499 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:47,501 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:49,369 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:10:49,370 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:49,503 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:51,505 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:53,508 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:10:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:10:54,833 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:10:54,945 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:10:55,510 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:57,512 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:10:59,514 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:00,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:01,516 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:03,518 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:05,520 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:05,858 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:07,522 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:09,524 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:11:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:11:09,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:11:11,305 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:11,526 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:13,528 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:15,530 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:16,763 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:17,532 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:19,383 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:11:19,534 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:21,237 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:11:21,239 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:11:21,239 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:11:21,241 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:11:21,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:21,537 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:11:22,242 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:23,539 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:11:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:11:24,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:11:25,541 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:27,543 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:27,657 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:29,547 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:31,549 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:33,129 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:33,551 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:35,553 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:37,555 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:38,613 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:39,557 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:11:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:11:39,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:11:41,559 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:43,561 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:44,117 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:45,564 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:47,566 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:49,373 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:11:49,376 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:49,568 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:51,570 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:53,572 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:54,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:11:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:11:54,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:11:54,958 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:11:55,574 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:57,576 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:11:59,578 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:00,585 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:01,580 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:03,583 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:05,585 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:06,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:06,912 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:12:06,913 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:12:06,914 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:12:06,916 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:12:07,587 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:07,588 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:12:09,590 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:09,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:12:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:12:09,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:12:11,527 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:11,592 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:13,594 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:15,596 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:17,016 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:17,598 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:19,375 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:12:19,600 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:21,603 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:22,504 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:23,605 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:24,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:12:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:12:24,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:12:25,607 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:27,609 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:28,000 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:29,611 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:31,613 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:33,498 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:33,615 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:35,617 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:37,619 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:39,006 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:39,621 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:39,829 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:12:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:12:39,830 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:12:41,623 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:43,625 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:44,541 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:45,627 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:47,629 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:49,377 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:12:49,631 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:50,379 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:51,633 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:52,798 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:12:52,800 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:12:52,801 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:12:52,804 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:12:53,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:53,636 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:12:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:12:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:12:54,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:12:55,638 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:55,985 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:12:57,640 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:12:59,642 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:01,099 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:01,644 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:03,646 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:05,648 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:06,584 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:07,650 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:09,652 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:13:09,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:13:09,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:13:11,654 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:12,072 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:13,656 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:15,658 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:17,563 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:17,660 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:19,392 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:13:19,662 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:21,665 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:23,059 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:23,666 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:13:24,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:13:24,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:13:25,668 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:27,671 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:28,560 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:29,673 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:31,675 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:33,677 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:34,068 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:35,679 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:37,681 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:38,602 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:13:38,604 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:13:38,604 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:13:38,606 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:13:38,683 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:13:39,607 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:39,684 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:13:39,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:13:39,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:13:41,686 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:43,688 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:45,048 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:45,690 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:47,692 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:49,382 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:13:49,694 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:50,540 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:51,696 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:53,698 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:13:54,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:13:54,831 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:13:55,700 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:56,169 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:13:57,702 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:13:59,704 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:01,624 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:01,706 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:03,708 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:05,710 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:07,093 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:07,712 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:09,714 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:09,856 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:14:09,857 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:14:09,864 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:14:11,716 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:12,582 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:13,719 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:15,721 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:17,723 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:18,074 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:19,384 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:14:19,725 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:21,730 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:23,554 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:23,731 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:24,421 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:14:24,423 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:14:24,423 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:14:24,425 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:14:24,733 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:14:24,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:14:24,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:14:24,844 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:14:25,734 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:27,736 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:29,051 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:29,738 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:31,740 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:33,742 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:34,517 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:35,744 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:37,746 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:39,748 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:39,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:14:39,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:14:39,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:14:39,956 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:41,750 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:43,752 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:45,425 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:45,755 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:47,757 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:49,385 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:14:49,760 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:50,888 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:51,762 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:53,764 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:54,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:14:54,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:14:54,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:14:55,766 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:56,325 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:14:57,769 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:14:59,771 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:01,773 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:01,784 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:03,775 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:05,777 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:07,265 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:07,779 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:09,781 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:09,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:15:09,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:15:09,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:15:09,943 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:15:09,943 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:15:09,944 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:15:09,945 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:15:10,783 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:15:11,784 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:12,744 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:13,786 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:15,788 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:17,791 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:18,221 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:19,388 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:15:19,793 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:21,796 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:23,703 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:23,798 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:24,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:15:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:15:24,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:15:25,800 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:27,803 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:29,187 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:29,806 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:31,808 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:33,810 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:34,669 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:35,813 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:37,814 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:39,816 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:39,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:15:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:15:39,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:15:39,978 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:41,819 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:43,820 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:45,571 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:45,822 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:47,824 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:49,390 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:15:49,826 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:51,057 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:51,829 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:53,831 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:54,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:15:54,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:15:54,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:15:55,586 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:15:55,587 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:15:55,587 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:15:55,589 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:15:55,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:55,834 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:15:56,589 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:15:57,836 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:15:59,838 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:01,840 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:02,021 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:03,842 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:05,846 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:07,481 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:07,848 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:09,844 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:16:09,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:16:09,845 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:16:09,850 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:11,852 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:12,921 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:13,855 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:15,857 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:17,859 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:18,378 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:19,393 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:16:19,861 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:21,863 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:23,826 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:23,867 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:16:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:16:24,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:16:25,868 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:27,871 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:29,328 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:29,873 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:31,875 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:33,877 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:34,804 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:35,879 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:37,881 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:16:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:16:39,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:16:39,883 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:40,005 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:41,126 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:16:41,127 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:16:41,127 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:16:41,129 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:16:41,886 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:41,887 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:16:42,710 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:16:42,713 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 16:16:42,714 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 16:16:42,714 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 16:16:42,715 DEBUG SenderThread:156911 [sender.py:send():379] send: metric +2024-04-25 16:16:42,715 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:16:42,715 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:16:42,718 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:16:42,888 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:16:43,889 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:45,727 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:47,894 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:49,395 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:16:51,397 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:54,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:16:54,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:16:54,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:16:56,472 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:16:57,905 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:16:59,908 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:01,910 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:01,926 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:02,912 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/config.yaml +2024-04-25 16:17:03,913 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:05,915 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:07,389 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:07,917 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:09,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:17:09,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:17:09,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:17:09,919 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:11,926 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:12,848 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:13,928 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:15,930 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:17,932 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:18,340 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:19,418 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:17:19,934 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:21,936 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:23,830 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:24,011 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:24,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:17:24,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:17:24,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:17:26,013 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:28,028 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:29,317 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:30,039 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:32,112 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:34,118 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:34,793 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:36,129 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:38,131 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:39,845 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-25 16:17:39,846 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: stop_status +2024-04-25 16:17:39,846 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: stop_status +2024-04-25 16:17:39,947 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:40,151 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:42,102 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: partial_history +2024-04-25 16:17:42,104 DEBUG SenderThread:156911 [sender.py:send():379] send: history +2024-04-25 16:17:42,104 DEBUG SenderThread:156911 [sender.py:send_request():406] send_request: summary_record +2024-04-25 16:17:42,106 INFO SenderThread:156911 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-25 16:17:42,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:42,154 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json +2024-04-25 16:17:44,156 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:45,709 DEBUG HandlerThread:156911 [handler.py:handle_request():146] handle_request: status_report +2024-04-25 16:17:46,158 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:48,160 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log +2024-04-25 16:17:49,399 DEBUG SenderThread:156911 [sender.py:send():379] send: stats +2024-04-25 16:17:50,162 INFO Thread-12 :156911 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240425_134518-etajcxpg/files/output.log